level=info ts=2026-05-04T11:51:49.47752011Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T11:51:49.477893957Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-04T11:51:49.478929539Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T11:51:49.479333546Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T11:51:49.479428614Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.59 inf=eth0
level=debug ts=2026-05-04T11:51:49.480317159Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T11:51:49.480378204Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.59 inf=eth0
level=info ts=2026-05-04T11:51:49.480402099Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-6fcd9c02
ts=2026-05-04T11:51:49.480466695Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T11:51:49.480478204Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.59 advertisePort=7946
level=debug ts=2026-05-04T11:51:49.480499951Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.59 advertisePort=7946
level=info ts=2026-05-04T11:51:49.480613663Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-04T11:51:49.48883032Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T11:51:49.488846846Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:51:49.488856652Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=8.244569ms
level=error ts=2026-05-04T11:51:49.488865599Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T11:51:49.48887739Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T11:51:49.491393614Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:49.491406906Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T11:51:49.495553512Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-04T11:51:49.495542967Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-04T11:51:49.495573521Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-04T11:51:49.495587382Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-04T11:51:49.495612955Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-04T11:51:49.495674439Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-04T11:51:49.495706636Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-04T11:51:49.495747654Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-04T11:51:49.495800773Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-04T11:51:49.495806352Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-04T11:51:49.495839095Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-04T11:51:49.495866174Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-04T11:51:49.495885305Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-04T11:51:49.496062831Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-04T11:51:49.496081441Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-04T11:51:49.496109448Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-04T11:51:50.496658721Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-04T11:51:50.628940762Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-04T11:51:50.629003109Z caller=loki.go:615 msg="Loki started" startup_time=1.215637152s
ts=2026-05-04T11:51:51.332157184Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:51.33220292Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:51:54.699223977Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:54.699252046Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T11:51:55.629441943Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-04T11:51:55.629491873Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-04T11:51:59.43915519Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:59.439172898Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:52:05.784080347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41772"
level=debug ts=2026-05-04T11:52:06.349151339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:52:07.422223255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:47014"
ts=2026-05-04T11:52:12.039181192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:42954"
ts=2026-05-04T11:52:14.984449742Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.55:7946: dial tcp 10.128.0.55:7946: connect: connection refused"
ts=2026-05-04T11:52:14.985485669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.56:7946"
ts=2026-05-04T11:52:14.986942924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.59:7946"
ts=2026-05-04T11:52:14.987074371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53474"
ts=2026-05-04T11:52:14.987597596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.56:7946"
ts=2026-05-04T11:52:14.990312586Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.58:7946: dial tcp 10.129.0.58:7946: connect: connection refused"
level=info ts=2026-05-04T11:52:14.990327969Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=25.501443857s
ts=2026-05-04T11:52:23.759979565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:35502"
level=debug ts=2026-05-04T11:52:27.55934365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:52:46.309421942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:52:49.775433672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T11:53:16.209516071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:53:16.312047576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T11:53:46.3144916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:53:47.528557666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T11:53:47.530049088Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T11:54:16.317685916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:54:23.133595844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:54:24.995929047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56646"
ts=2026-05-04T11:54:46.320290147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T11:54:54.997016792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:42412"
ts=2026-05-04T11:55:16.322620068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T11:55:23.780941136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42488"
level=debug ts=2026-05-04T11:55:40.044047036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:55:46.325045879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T11:55:47.512262634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:33414"
ts=2026-05-04T11:56:16.325995958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T11:56:17.513727531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43062"
ts=2026-05-04T11:56:46.327318404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T11:56:53.791178264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59934"
level=debug ts=2026-05-04T11:56:57.924787438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:57:16.329868424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T11:57:46.333150018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T11:57:47.519801789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43520"
ts=2026-05-04T11:57:53.796928336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:60628"
level=debug ts=2026-05-04T11:58:08.525943785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:58:16.335245607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T11:58:23.800387375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:54018"
ts=2026-05-04T11:58:46.336420728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T11:58:47.524273772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:36492"
ts=2026-05-04T11:58:53.803553384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56342"
ts=2026-05-04T11:59:16.338468656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:59:26.844023694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T11:59:26.845906469Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T11:59:46.341334447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T11:59:53.810272847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:33478"
ts=2026-05-04T12:00:16.343581307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:00:23.813984561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56734"
level=debug ts=2026-05-04T12:00:26.214238714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:00:46.345571034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:01:04.165066818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:01:16.347523556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:01:17.537860138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60374"
ts=2026-05-04T12:01:23.819559387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:49362"
ts=2026-05-04T12:01:46.349682961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:01:55.0307182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:59134"
level=info ts=2026-05-04T12:01:55.629506742Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-04T12:01:55.633088769Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:01:55.635299072Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:01:55.637313417Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:01:55.637338453Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T12:01:55.637352113Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-04T12:02:05.015082095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:02:16.352330175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:02:17.540213172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40290"
level=debug ts=2026-05-04T12:02:40.515172367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:02:46.354846819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:02:55.033151693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:45720"
ts=2026-05-04T12:03:16.35589843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:03:36.369406568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:03:36.37140033Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:03:46.357146169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:04:16.358677149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:04:21.521147323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:04:23.838780374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36324"
ts=2026-05-04T12:04:25.040338904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:46036"
ts=2026-05-04T12:04:46.360836609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:05:04.322407258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:05:16.365012876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:05:45.510779116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:05:46.366500695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:05:47.554136342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40822"
ts=2026-05-04T12:06:16.36765248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:06:23.851288979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:43542"
level=debug ts=2026-05-04T12:06:32.909908055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:06:46.371504706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:06:53.854458589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59960"
ts=2026-05-04T12:07:16.377561379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:07:23.414719488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:07:23.417592797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:07:23.857765711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36138"
ts=2026-05-04T12:07:46.381064255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:07:47.561485858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51638"
ts=2026-05-04T12:07:53.860641461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56352"
ts=2026-05-04T12:08:16.384960582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:08:17.563420228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42724"
level=debug ts=2026-05-04T12:08:19.206554069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:08:23.863882378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52888"
ts=2026-05-04T12:08:25.058453139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57296"
ts=2026-05-04T12:08:46.387896847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:08:53.867157143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:33568"
ts=2026-05-04T12:08:55.059308293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:51452"
level=debug ts=2026-05-04T12:09:00.961156923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:09:16.39053947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:09:23.869907425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59608"
ts=2026-05-04T12:09:46.392594259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:09:47.566891542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:34528"
ts=2026-05-04T12:09:53.873852979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59452"
ts=2026-05-04T12:09:55.064327227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:46198"
level=debug ts=2026-05-04T12:09:56.857498344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:10:16.395455379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:10:25.065864166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55628"
ts=2026-05-04T12:10:46.397938483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:10:50.277802343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:10:55.067025419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:58218"
ts=2026-05-04T12:11:16.399646322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:11:25.067950334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39658"
level=debug ts=2026-05-04T12:11:38.2575947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:11:38.259643755Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:11:46.401251497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:11:53.888905729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:51174"
level=error ts=2026-05-04T12:11:55.642848125Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:11:55.645621595Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:11:55.648378854Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:11:55.648410414Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:12:16.405698401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:12:17.57576512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42648"
ts=2026-05-04T12:12:23.892351913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:43384"
level=debug ts=2026-05-04T12:12:38.58934971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:12:46.408838751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:12:47.577405151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54518"
ts=2026-05-04T12:12:55.073997112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:36194"
ts=2026-05-04T12:13:16.414590214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:13:25.075656397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43532"
level=debug ts=2026-05-04T12:13:32.462008261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:13:46.418056591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:13:53.907387337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:58750"
ts=2026-05-04T12:14:16.420498416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:14:22.778736989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:14:46.421450712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:14:47.584358322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:59512"
ts=2026-05-04T12:14:53.915455811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:43890"
level=debug ts=2026-05-04T12:15:10.361489409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:15:16.422584857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:15:23.918819176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:57334"
ts=2026-05-04T12:15:46.424320717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:15:55.083555581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56844"
level=debug ts=2026-05-04T12:16:08.09416477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:16:08.096108321Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:16:16.426701457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:16:25.085277782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:33716"
ts=2026-05-04T12:16:46.429623016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:16:54.448458047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:16:55.086416615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:58876"
ts=2026-05-04T12:17:16.43133236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:17:17.596026449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44556"
ts=2026-05-04T12:17:25.087548079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:52936"
ts=2026-05-04T12:17:46.432576494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:17:53.935688786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36120"
level=debug ts=2026-05-04T12:17:53.95161499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:18:16.435346569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:18:46.437939127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:18:47.397012689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:18:53.942218259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:57794"
ts=2026-05-04T12:18:55.093158916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57824"
ts=2026-05-04T12:19:16.440749582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:19:17.612327936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51346"
ts=2026-05-04T12:19:23.945659403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59982"
level=debug ts=2026-05-04T12:19:46.169008326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:19:46.442997079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:19:55.097764025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:44822"
ts=2026-05-04T12:20:16.444761811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:20:23.952331865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:38712"
level=debug ts=2026-05-04T12:20:41.195965742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:20:41.198220866Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:20:46.447568328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:20:47.618972066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:53148"
ts=2026-05-04T12:21:16.449723396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:21:19.64573759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:21:23.959178564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:55362"
ts=2026-05-04T12:21:46.45182976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=error ts=2026-05-04T12:21:55.642749867Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:21:55.645156755Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:21:55.647458945Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:21:55.647480406Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:22:09.894533525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:22:16.454142176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:22:45.464016999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:22:46.456927074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:23:16.460568765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:23:23.971469672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36036"
level=debug ts=2026-05-04T12:23:40.283424369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:23:46.462504259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:23:47.635499067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37520"
ts=2026-05-04T12:24:16.463816381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:24:19.669821604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:24:19.671415996Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:24:46.465484729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:24:55.118480831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41974"
ts=2026-05-04T12:25:16.466750933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:25:17.640591518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50726"
level=debug ts=2026-05-04T12:25:19.534491632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:25:23.983375346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:38062"
ts=2026-05-04T12:25:25.119407689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:33432"
ts=2026-05-04T12:25:46.467575785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:25:53.985792075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:50198"
level=debug ts=2026-05-04T12:26:07.521474445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:26:16.469678932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:26:17.64259811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56904"
ts=2026-05-04T12:26:46.472502056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:26:47.200005961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:27:16.474399337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:27:17.646985774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:35522"
level=debug ts=2026-05-04T12:27:42.321562299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:27:46.476170952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:27:47.648656509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:45048"
ts=2026-05-04T12:28:16.479461039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:28:17.649668108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40544"
ts=2026-05-04T12:28:25.133477206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:34380"
level=debug ts=2026-05-04T12:28:37.55254725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:28:37.554583072Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:28:46.481897906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:29:16.484212272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:29:31.501151344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:29:46.486092308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:30:08.688348921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:30:16.487729004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:30:24.016444782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:51076"
ts=2026-05-04T12:30:25.141527137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:37092"
ts=2026-05-04T12:30:46.490281189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:31:07.345939684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:31:16.492234407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:31:17.664554963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49990"
ts=2026-05-04T12:31:46.494402746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:31:49.070393267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:31:55.640426042Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:31:55.642299122Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:31:55.644003323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:31:55.64403058Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:32:16.497998008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:32:25.148743181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:48866"
level=debug ts=2026-05-04T12:32:43.732469661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:32:43.734066216Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:32:46.499962294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:33:16.501616136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:33:37.645505458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:33:46.504928152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:33:47.67173795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60908"
ts=2026-05-04T12:33:54.038748713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42714"
level=debug ts=2026-05-04T12:34:14.14885288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:34:16.508747979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:34:17.673355598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50886"
ts=2026-05-04T12:34:46.511381788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:35:15.601303741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:35:16.512292889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:35:46.51324871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:35:54.051409353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36354"
ts=2026-05-04T12:35:55.167903214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:48490"
level=debug ts=2026-05-04T12:36:14.13373162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:36:16.514785895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:36:17.681293483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:35900"
ts=2026-05-04T12:36:46.516146828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:37:14.131506071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:37:14.133634737Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:37:16.518206277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:37:17.685418828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:33486"
ts=2026-05-04T12:37:46.52131834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:38:05.325979906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:38:16.524288695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:38:24.067206252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:38280"
ts=2026-05-04T12:38:46.526848363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:38:47.689202687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49964"
level=debug ts=2026-05-04T12:38:56.903043862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:39:16.52903998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:39:24.07344329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41470"
ts=2026-05-04T12:39:46.53228176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:39:55.182686667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:50432"
level=debug ts=2026-05-04T12:39:57.243848792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:40:16.534203381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:40:17.697747868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:33658"
ts=2026-05-04T12:40:25.184461339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:50570"
level=debug ts=2026-05-04T12:40:37.605435384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:40:46.536232742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:40:47.698816158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43998"
ts=2026-05-04T12:40:54.083362894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:53816"
level=debug ts=2026-05-04T12:41:14.939990949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:41:14.941688199Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:41:16.5396096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:41:17.700164261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:58996"
ts=2026-05-04T12:41:46.542775156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:41:47.70155853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43884"
level=error ts=2026-05-04T12:41:55.641738813Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:41:55.644089953Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:41:55.646463448Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:41:55.646491276Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:42:06.132425448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:42:16.545136724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:42:17.703384922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:45908"
ts=2026-05-04T12:42:25.193520757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:55478"
ts=2026-05-04T12:42:46.54597886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:42:56.092040535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:43:16.548956146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:43:24.099714059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:38768"
level=debug ts=2026-05-04T12:43:44.872995643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:43:46.552142157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:43:54.102970554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46888"
ts=2026-05-04T12:44:16.555110685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:44:33.184913982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:44:46.556823204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:44:47.709791573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37452"
level=debug ts=2026-05-04T12:45:08.748304478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:45:08.74991602Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:45:16.558619218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:45:46.560330735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:45:57.904899763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:46:16.561547794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:46:17.715030851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54616"
ts=2026-05-04T12:46:24.118640257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59390"
ts=2026-05-04T12:46:46.563775939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:46:47.213579691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:47:16.566304339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:47:30.578695732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:47:46.568100967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:48:16.570432284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:48:18.401806466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:48:46.572451056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:48:54.13467356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56892"
ts=2026-05-04T12:48:55.219787382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:36630"
level=debug ts=2026-05-04T12:49:15.552771085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:49:15.554529046Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:49:16.574627523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:49:17.72224943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54148"
ts=2026-05-04T12:49:24.138033791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41878"
ts=2026-05-04T12:49:25.221606245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:60300"
ts=2026-05-04T12:49:46.576504014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:49:52.686293557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:50:16.578481512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:50:25.227080665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41420"
ts=2026-05-04T12:50:46.582598397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:50:47.726834306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42332"
level=debug ts=2026-05-04T12:50:53.709294681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:51:16.58521007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:51:17.727660214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:36844"
ts=2026-05-04T12:51:24.150816718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:44652"
level=debug ts=2026-05-04T12:51:40.359136174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:51:46.588852727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:51:55.234967238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43706"
level=error ts=2026-05-04T12:51:55.641573057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:51:55.644310947Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:51:55.646799923Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T12:51:55.646829509Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:52:16.592652448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:52:25.236436555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:38554"
level=debug ts=2026-05-04T12:52:36.683946074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:52:46.595342664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:52:47.73370768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38486"
ts=2026-05-04T12:52:54.160221501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52354"
ts=2026-05-04T12:52:55.237820145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41532"
ts=2026-05-04T12:53:16.596883386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:53:17.735290446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37610"
level=debug ts=2026-05-04T12:53:18.578010655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:53:18.579853643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:53:25.239142571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49902"
ts=2026-05-04T12:53:46.5993163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:53:56.664335837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:54:16.601480239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:54:25.242487943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:41788"
ts=2026-05-04T12:54:46.603099426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:54:51.824096683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:54:54.17269162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41474"
ts=2026-05-04T12:54:55.243732455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49040"
ts=2026-05-04T12:55:16.604640951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
level=debug ts=2026-05-04T12:55:27.068842005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:55:46.606080122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:55:55.248546826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:54532"
ts=2026-05-04T12:56:16.607971766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:56:27.788487451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:56:46.609665086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:57:16.611087955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:57:25.25195988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:32904"
level=debug ts=2026-05-04T12:57:29.798705737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:57:29.800264408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T12:57:46.612839422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:57:47.756767155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54338"
ts=2026-05-04T12:57:55.25385873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:40706"
level=debug ts=2026-05-04T12:58:14.300377771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:58:16.613704229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:58:24.195953798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:39830"
ts=2026-05-04T12:58:46.6160625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T12:58:55.25828046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:43716"
level=debug ts=2026-05-04T12:58:57.304754688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:59:16.618440302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T12:59:17.759997412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60834"
ts=2026-05-04T12:59:25.260033904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:45540"
level=debug ts=2026-05-04T12:59:39.385553305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:59:46.619846596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T12:59:47.761791305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:45822"
ts=2026-05-04T12:59:54.205387358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36842"
level=debug ts=2026-05-04T13:00:14.532715283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:00:16.62133763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:00:46.622848015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:00:55.266210026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49200"
level=debug ts=2026-05-04T13:00:59.234532505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:00:59.236410921Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T13:01:16.624738442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:01:24.215350787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59026"
level=debug ts=2026-05-04T13:01:41.657968716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:01:46.627042476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:01:54.218272791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:32908"
level=error ts=2026-05-04T13:01:55.641796837Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:01:55.644312648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:01:55.646581708Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:01:55.646618382Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T13:02:16.630769732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:02:19.570372054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:02:46.633802491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:02:47.772075777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:36764"
level=debug ts=2026-05-04T13:03:14.828331517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:03:16.635763093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:03:46.637872689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:03:51.653466765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:03:54.231357765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42632"
ts=2026-05-04T13:04:16.640255088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:04:46.642766308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:04:51.653255342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:04:51.654837534Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T13:04:55.285725108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:56818"
ts=2026-05-04T13:05:16.645022628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:05:17.783588887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56572"
ts=2026-05-04T13:05:46.646112871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:05:47.784651788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38290"
level=debug ts=2026-05-04T13:05:50.616360537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:06:16.64857167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:06:46.652011764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:06:47.163685392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:06:55.297159152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:51810"
ts=2026-05-04T13:07:16.654178418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:07:17.790281616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:52242"
level=debug ts=2026-05-04T13:07:44.718574514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:07:46.656285875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:07:54.256323812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:53264"
ts=2026-05-04T13:08:16.659286644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:08:21.443491013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:08:46.661227646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
ts=2026-05-04T13:08:55.305054898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:58048"
level=debug ts=2026-05-04T13:09:05.959009421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:09:05.96078414Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T13:09:16.663856986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:09:17.797424113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:46608"
ts=2026-05-04T13:09:24.265092129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:60704"
level=debug ts=2026-05-04T13:09:43.236754243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:09:46.667271905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
ts=2026-05-04T13:10:16.669312777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:10:25.313100984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:32768"
level=debug ts=2026-05-04T13:10:37.546579039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:10:46.670510906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:10:47.805101064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43928"
ts=2026-05-04T13:10:54.274860851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:58284"
ts=2026-05-04T13:10:55.314612917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:49736"
ts=2026-05-04T13:11:16.673166722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:11:16.860752684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:11:17.806746654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:52044"
ts=2026-05-04T13:11:25.315713791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:50414"
ts=2026-05-04T13:11:46.67571228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:11:54.280879745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42932"
ts=2026-05-04T13:11:55.317479574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:57684"
level=error ts=2026-05-04T13:11:55.641579923Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:11:55.644023041Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:11:55.646503867Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-04T13:11:55.646529643Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T13:12:09.089084045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:12:16.677175667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:12:25.319099689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:58956"
level=debug ts=2026-05-04T13:12:44.316253881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:12:44.317821837Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-04T13:12:46.678472956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:13:16.679581467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:13:39.69531314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:13:46.681426486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:14:16.684171463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:14:22.872645905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:14:46.686990985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:15:13.477659669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:15:16.689045091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1d2a49ff 10.129.0.60:7946"
ts=2026-05-04T13:15:17.826064192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:46168"
ts=2026-05-04T13:15:25.334413462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.60:39046"