level=info ts=2026-04-28T22:53:17.430440128Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-28T22:53:17.430462639Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-28T22:53:17.431261768Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T22:53:17.431574229Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T22:53:17.431638984Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.79 inf=eth0
level=debug ts=2026-04-28T22:53:17.43254352Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T22:53:17.432603693Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.79 inf=eth0
level=info ts=2026-04-28T22:53:17.432866403Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-8caeade0
ts=2026-04-28T22:53:17.432952812Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T22:53:17.43296578Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.79 advertisePort=7946
level=debug ts=2026-04-28T22:53:17.432984964Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.79 advertisePort=7946
level=info ts=2026-04-28T22:53:17.43310479Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T22:53:17.440694484Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T22:53:17.440713991Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T22:53:17.440725613Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.622947ms
level=error ts=2026-04-28T22:53:17.440734637Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-28T22:53:17.440746975Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-28T22:53:17.443109095Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T22:53:17.443122465Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T22:53:17.447100435Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-28T22:53:17.447104058Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-28T22:53:17.447111153Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-28T22:53:17.447133864Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-28T22:53:17.44713082Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-28T22:53:17.447219753Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T22:53:17.447464887Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-28T22:53:17.447474728Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-28T22:53:17.447478171Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-28T22:53:17.4474815Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-28T22:53:17.447552801Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-28T22:53:17.447560847Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-28T22:53:17.447577463Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T22:53:17.447729662Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-28T22:53:17.447738684Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-28T22:53:17.447748527Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-28T22:53:18.448345846Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-28T22:53:18.610649063Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-28T22:53:18.610698995Z caller=loki.go:615 msg="Loki started" startup_time=1.244107153s
ts=2026-04-28T22:53:18.939422827Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T22:53:18.939445212Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:53:22.357531786Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T22:53:22.357562975Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T22:53:23.610952794Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-28T22:53:23.611003776Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-28T22:53:28.137303645Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T22:53:28.137325914Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T22:53:30.882435851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:53:35.719030926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52868"
ts=2026-04-28T22:53:36.911454593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48696"
ts=2026-04-28T22:53:37.877480315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49364"
ts=2026-04-28T22:53:43.852298963Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.80:7946: dial tcp 10.130.0.80:7946: connect: connection refused"
ts=2026-04-28T22:53:43.852416793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.79:7946"
ts=2026-04-28T22:53:43.852460093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54264"
ts=2026-04-28T22:53:43.8550825Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-04-28T22:53:43.856459053Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.82:7946: dial tcp 10.129.0.82:7946: connect: connection refused"
ts=2026-04-28T22:53:43.857505147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-28T22:53:43.859930824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
level=info ts=2026-04-28T22:53:43.861475014Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.420722869s
level=debug ts=2026-04-28T22:53:50.3676676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:54:13.379645108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T22:54:14.654442212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:54:19.910732297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45196"
ts=2026-04-28T22:54:35.801173163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58542"
ts=2026-04-28T22:54:43.383438533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T22:54:46.572692562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:55:00.739917639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47092"
ts=2026-04-28T22:55:05.804616618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49022"
ts=2026-04-28T22:55:13.386167734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T22:55:17.389581314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T22:55:17.39133316Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:55:43.388655878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T22:55:54.020854477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:56:00.744539303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50074"
ts=2026-04-28T22:56:05.809285728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39964"
ts=2026-04-28T22:56:13.391640874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T22:56:42.357066219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:56:43.394901826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T22:56:49.920999122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54164"
ts=2026-04-28T22:57:13.397449714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T22:57:30.751833563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59438"
ts=2026-04-28T22:57:35.814852608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42506"
ts=2026-04-28T22:57:43.400205853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T22:57:47.69361452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:58:00.754541124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59490"
ts=2026-04-28T22:58:13.403059181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T22:58:19.927950233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51572"
ts=2026-04-28T22:58:43.405702862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T22:58:55.563432089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T22:59:05.821855271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57262"
ts=2026-04-28T22:59:13.409267071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T22:59:35.825381508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59060"
ts=2026-04-28T22:59:43.41248332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:00:05.828089074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60042"
level=debug ts=2026-04-28T23:00:07.21495704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T23:00:07.216747963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:00:13.415451744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:00:43.418748685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:00:57.885009371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:01:13.421636316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:01:19.939441684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56378"
ts=2026-04-28T23:01:35.835565754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38814"
ts=2026-04-28T23:01:43.425159781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:01:45.872370679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:02:13.428609397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:02:29.26441958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:02:30.774893796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60112"
ts=2026-04-28T23:02:35.841286375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58490"
ts=2026-04-28T23:02:43.431318825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:03:00.777985937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46928"
level=debug ts=2026-04-28T23:03:13.318245765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:03:13.434871971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=info ts=2026-04-28T23:03:23.611653452Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-28T23:03:23.616522437Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:03:23.618169709Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:03:23.61982821Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:03:23.619850835Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T23:03:23.619862889Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-28T23:03:35.846383756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33306"
ts=2026-04-28T23:03:43.438024848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:04:05.849374682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39798"
level=debug ts=2026-04-28T23:04:06.525649833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:04:06.527164199Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:04:13.440822813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:04:19.951410292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50820"
level=debug ts=2026-04-28T23:04:42.709488574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:04:43.444233698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:05:13.446887954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:05:35.855303634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59012"
level=debug ts=2026-04-28T23:05:38.994621187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:05:43.450072462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:05:49.957407252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57790"
ts=2026-04-28T23:06:13.453706893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:06:35.861513856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55422"
level=debug ts=2026-04-28T23:06:39.899264366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:06:43.456861448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:07:00.795952519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45092"
ts=2026-04-28T23:07:13.459728598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:07:14.411908976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:07:30.799200424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34846"
ts=2026-04-28T23:07:43.462359979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:07:49.965560547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45870"
level=debug ts=2026-04-28T23:08:08.066100815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:08:08.067787299Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:08:13.465078413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:08:19.968925782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57280"
ts=2026-04-28T23:08:35.870097092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41836"
ts=2026-04-28T23:08:43.468713549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:08:59.769243632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:09:00.807143399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36390"
ts=2026-04-28T23:09:13.471950699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:09:43.475230813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:09:48.155363866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:10:05.876717131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43310"
ts=2026-04-28T23:10:13.477872304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:10:30.814466363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50862"
level=debug ts=2026-04-28T23:10:41.734545938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:10:43.481446972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:11:13.484239901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:11:30.820190162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51090"
level=debug ts=2026-04-28T23:11:35.649142298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:11:43.48777037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:12:13.490715462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:12:19.986958099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45758"
ts=2026-04-28T23:12:30.826256642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35828"
level=debug ts=2026-04-28T23:12:33.858110589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:12:33.860270715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:12:43.493670894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:13:12.926808894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:13:13.496941267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=error ts=2026-04-28T23:13:23.62536152Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:13:23.627894979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:13:23.630438583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:13:23.630472347Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:13:43.499522763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:13:56.877164365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:14:13.503330313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:14:19.996352331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58584"
ts=2026-04-28T23:14:30.83587006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:33538"
ts=2026-04-28T23:14:43.505968456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:14:46.342904253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:14:49.999306718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33064"
ts=2026-04-28T23:15:05.899929523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58498"
ts=2026-04-28T23:15:13.509419929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:15:20.002124483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50784"
level=debug ts=2026-04-28T23:15:30.865692769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:15:43.512486259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:16:00.842212498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38100"
ts=2026-04-28T23:16:05.905498434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39120"
ts=2026-04-28T23:16:13.515460947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:16:17.156293967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:16:17.158006007Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:16:30.846278676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34630"
ts=2026-04-28T23:16:43.518345509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:16:50.012092342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55008"
level=debug ts=2026-04-28T23:17:02.24982416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:17:13.521470593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:17:20.015372868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53350"
ts=2026-04-28T23:17:30.851912693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34326"
level=debug ts=2026-04-28T23:17:39.925303673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:17:43.52586022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:18:00.85536718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44826"
ts=2026-04-28T23:18:13.529596153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:18:20.019807343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46170"
ts=2026-04-28T23:18:30.858359961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56794"
level=debug ts=2026-04-28T23:18:36.154826097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:18:43.532797983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:18:50.022524354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53206"
ts=2026-04-28T23:19:13.541691975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:19:29.163451798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:19:43.545962373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:20:09.955655525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:20:09.957785875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:20:13.554157388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:20:20.029693397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58468"
ts=2026-04-28T23:20:43.556735605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:20:52.098246714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:21:13.559510029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:21:43.562386155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:21:52.421143279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:22:13.565902265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:22:29.180369016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:22:30.878319972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54148"
ts=2026-04-28T23:22:35.929117339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52618"
ts=2026-04-28T23:22:43.56964034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:22:50.041714158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50626"
ts=2026-04-28T23:23:00.88146248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37016"
level=debug ts=2026-04-28T23:23:08.784099871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:23:13.572949798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=error ts=2026-04-28T23:23:23.624361979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:23:23.626416354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:23:23.628451172Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:23:23.628481352Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:23:43.576143633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:23:55.383174403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:23:55.384899451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:24:13.579128912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:24:20.04893806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43494"
ts=2026-04-28T23:24:30.888596471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34146"
ts=2026-04-28T23:24:35.937888675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36532"
ts=2026-04-28T23:24:43.582243797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:24:45.383687042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:25:13.585193636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:25:28.655715947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:25:43.5883312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:25:50.054905173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37734"
ts=2026-04-28T23:26:00.9119391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39960"
ts=2026-04-28T23:26:05.94460048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40604"
ts=2026-04-28T23:26:13.591202516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:26:13.987089314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:26:20.058552496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36258"
ts=2026-04-28T23:26:30.916129639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60118"
ts=2026-04-28T23:26:35.947596756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35452"
ts=2026-04-28T23:26:43.594198056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:26:59.705279322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:27:05.951079867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:34288"
ts=2026-04-28T23:27:13.597142691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:27:20.06268775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36806"
ts=2026-04-28T23:27:43.6004642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:27:50.065651956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46774"
level=debug ts=2026-04-28T23:27:57.256486816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:27:57.258087859Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:28:00.924345227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59742"
ts=2026-04-28T23:28:05.956276005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46728"
ts=2026-04-28T23:28:13.603988465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:28:20.068258599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55768"
ts=2026-04-28T23:28:30.92742181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55480"
level=debug ts=2026-04-28T23:28:40.672413896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:28:43.607401891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:28:50.071602849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43566"
ts=2026-04-28T23:29:13.609984869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:29:22.452418186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:29:30.933614778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:36076"
ts=2026-04-28T23:29:43.613584195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:29:50.077390151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40166"
ts=2026-04-28T23:30:00.937161385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:52204"
ts=2026-04-28T23:30:05.96634013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48304"
ts=2026-04-28T23:30:13.617379514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:30:15.916429074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:30:43.620217463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:30:50.082687415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60276"
ts=2026-04-28T23:31:05.970945572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48836"
level=debug ts=2026-04-28T23:31:06.160322419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:31:13.623784857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:31:43.626394706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:31:45.531862316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:31:45.533576285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:32:00.948911325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56880"
ts=2026-04-28T23:32:13.630083631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:32:20.090538197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42852"
ts=2026-04-28T23:32:30.952233089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42456"
ts=2026-04-28T23:32:43.633480171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:32:44.236776672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:33:13.637354836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:33:20.095245795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58246"
level=debug ts=2026-04-28T23:33:21.783051794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:33:23.623091383Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:33:23.624664325Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:33:23.626726069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:33:23.626750388Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:33:30.956708916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:46998"
ts=2026-04-28T23:33:43.640718478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:34:05.983200067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48440"
ts=2026-04-28T23:34:13.644213827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:34:22.058506764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:34:43.647495766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:34:56.887921116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:35:00.964967948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45964"
ts=2026-04-28T23:35:05.988583432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38428"
ts=2026-04-28T23:35:13.650337858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:35:37.301072461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:35:37.302907585Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:35:43.654091483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:36:00.970749653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:57894"
ts=2026-04-28T23:36:05.99397417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56856"
ts=2026-04-28T23:36:13.657269686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:36:20.10633976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43350"
level=debug ts=2026-04-28T23:36:25.414113515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:36:43.659956777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:37:00.978913097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59476"
level=debug ts=2026-04-28T23:37:11.480494282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:37:13.663239556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:37:30.9826096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45622"
ts=2026-04-28T23:37:36.001425645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47496"
ts=2026-04-28T23:37:43.666467878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:37:50.113055854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40266"
ts=2026-04-28T23:38:06.004592149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48554"
level=debug ts=2026-04-28T23:38:06.284063902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:38:13.66991647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:38:41.387387124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:38:43.67323666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:39:13.676045741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:39:29.291586504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:39:29.293103498Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:39:30.992535015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40732"
ts=2026-04-28T23:39:43.679568671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:40:00.995361208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54976"
level=debug ts=2026-04-28T23:40:06.740522742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:40:13.682888031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:40:43.686371077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:40:50.126184313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35348"
level=debug ts=2026-04-28T23:40:55.975738557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:41:13.689335769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:41:20.13126866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47790"
ts=2026-04-28T23:41:31.004518931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51688"
level=debug ts=2026-04-28T23:41:41.052045719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:41:43.692379697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:42:13.695637038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:42:20.136069959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44316"
level=debug ts=2026-04-28T23:42:39.133250488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:42:43.698338384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:42:50.138846855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37932"
ts=2026-04-28T23:43:01.013196923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39210"
ts=2026-04-28T23:43:13.701824911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:43:15.437353318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:43:15.43901148Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:43:23.622562771Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:43:23.624064376Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:43:23.626195073Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:43:23.626218274Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:43:36.027965561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:34922"
ts=2026-04-28T23:43:43.704932525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:43:50.143728125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59636"
level=debug ts=2026-04-28T23:44:02.934420267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:44:13.708385905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:44:20.146583425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36180"
ts=2026-04-28T23:44:43.71121168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:45:00.610102903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:45:01.024020892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58084"
ts=2026-04-28T23:45:06.034563432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52434"
ts=2026-04-28T23:45:13.714726786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:45:43.718091143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:45:47.401405839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:45:50.154542992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51414"
ts=2026-04-28T23:46:13.720605676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:46:20.157676966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51546"
level=debug ts=2026-04-28T23:46:31.771027557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:46:43.724001142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:47:06.042223584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:34936"
ts=2026-04-28T23:47:13.72730606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:47:20.161813624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42108"
level=debug ts=2026-04-28T23:47:29.454559422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:47:29.455994319Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:47:36.045627939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40264"
ts=2026-04-28T23:47:43.730119421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:48:06.049020122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56738"
ts=2026-04-28T23:48:13.733719898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:48:20.310304249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:48:43.736987843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:49:03.995314846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:49:13.740596209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:49:43.743858114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:49:50.171923374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48304"
level=debug ts=2026-04-28T23:49:58.046140946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:50:01.046986877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55664"
ts=2026-04-28T23:50:13.747083896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:50:20.175356498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42724"
level=debug ts=2026-04-28T23:50:33.006326244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:50:36.060647213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:32842"
ts=2026-04-28T23:50:43.750357997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:51:01.053110646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45166"
ts=2026-04-28T23:51:13.753657775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:51:32.582074602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:51:32.583713693Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:51:43.75697027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:52:06.067571893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59046"
ts=2026-04-28T23:52:13.759984438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:52:31.061367338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35098"
level=debug ts=2026-04-28T23:52:32.624231498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:52:43.762909994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:53:06.0730067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39628"
ts=2026-04-28T23:53:13.765963252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:53:16.262720939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:53:20.186697427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60888"
level=error ts=2026-04-28T23:53:23.623193465Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:53:23.624859598Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:53:23.627019187Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T23:53:23.627041541Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:53:36.076432264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54756"
ts=2026-04-28T23:53:43.769588158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:54:01.719247983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:54:13.772711526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:54:31.070578744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48534"
ts=2026-04-28T23:54:43.776262265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:54:50.193954916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46452"
level=debug ts=2026-04-28T23:54:55.372575948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:55:01.073683331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55416"
ts=2026-04-28T23:55:13.77960049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:55:30.107730208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:55:30.109602559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:55:31.077067657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38478"
ts=2026-04-28T23:55:36.084943618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47366"
ts=2026-04-28T23:55:43.782613131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-28T23:56:06.088502432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40416"
ts=2026-04-28T23:56:13.785359809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-28T23:56:30.121190253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:56:43.788687969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:57:01.085572512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45722"
ts=2026-04-28T23:57:13.792287986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-28T23:57:30.498533414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:57:31.089129869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43270"
ts=2026-04-28T23:57:43.795538715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:58:13.798545057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:58:23.564719915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:58:36.10310334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50446"
ts=2026-04-28T23:58:43.80136247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-28T23:59:01.096977319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44898"
level=debug ts=2026-04-28T23:59:03.347413503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T23:59:13.804289665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-28T23:59:36.108842731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38920"
ts=2026-04-28T23:59:43.808086828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-28T23:59:45.497164034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T23:59:45.498801551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T23:59:50.214330121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33196"
ts=2026-04-29T00:00:13.811206717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:00:40.975579834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:00:43.813852182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:01:13.817476161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:01:31.11040399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34972"
level=debug ts=2026-04-29T00:01:33.879674388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:01:43.820593471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:01:50.222915715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58044"
ts=2026-04-29T00:02:13.823221212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:02:28.891015766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:02:43.826575134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:03:13.829234658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:03:16.350669686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:03:23.623717536Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:03:23.62526796Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:03:23.626777958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:03:23.626801145Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:03:31.119863087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34144"
ts=2026-04-29T00:03:43.832601314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:03:50.230977763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51676"
ts=2026-04-29T00:04:01.123005628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37716"
ts=2026-04-29T00:04:06.128129324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59610"
level=debug ts=2026-04-29T00:04:13.640242545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:04:13.641905867Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:04:13.835569448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:04:31.126279525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42054"
ts=2026-04-29T00:04:43.838183089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:04:50.235189056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46106"
level=debug ts=2026-04-29T00:05:01.196923659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:05:13.841720978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:05:20.238261913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59596"
ts=2026-04-29T00:05:36.135798406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45866"
ts=2026-04-29T00:05:43.845343321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:05:45.012460828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:06:01.134302833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45186"
ts=2026-04-29T00:06:06.138805358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45806"
ts=2026-04-29T00:06:13.848410284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:06:43.851470174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:06:45.391844811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:07:13.854453954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-29T00:07:31.753297867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:07:43.858186533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:07:50.246592441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:32986"
ts=2026-04-29T00:08:06.1458462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38480"
level=debug ts=2026-04-29T00:08:12.163144939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:08:12.164901572Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:08:13.861933329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:08:36.149055869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49378"
ts=2026-04-29T00:08:43.865564999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-29T00:09:03.854642815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:09:13.868784809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:09:43.871712289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:09:58.39196244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:10:01.150795988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38752"
ts=2026-04-29T00:10:13.875168751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:10:43.878263875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:10:52.753171295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:11:06.15841102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59342"
ts=2026-04-29T00:11:13.880882972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:11:20.262027094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37308"
level=debug ts=2026-04-29T00:11:29.652710776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:11:43.884658027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:12:01.160689194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38406"
ts=2026-04-29T00:12:06.162406576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50538"
ts=2026-04-29T00:12:13.887792872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:12:25.192983328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:12:25.194625483Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:12:31.163773649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:47842"
ts=2026-04-29T00:12:43.890556065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:12:50.269016847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46954"
ts=2026-04-29T00:13:13.893835127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:13:18.326949262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:13:23.6229672Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:13:23.624589423Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:13:23.626296672Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:13:23.626324449Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:13:36.168826756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49758"
ts=2026-04-29T00:13:43.896895813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:14:06.171838716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58818"
level=debug ts=2026-04-29T00:14:10.098799607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:14:13.899854838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:14:31.172698258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55728"
ts=2026-04-29T00:14:43.902672671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:14:55.027690447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:15:01.175356555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45832"
ts=2026-04-29T00:15:13.906125426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:15:31.178683378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49526"
level=debug ts=2026-04-29T00:15:38.915262575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:15:43.909062572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:16:13.912322597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:16:24.844899083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:16:24.84635226Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:16:43.914943495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:17:00.640791037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:17:13.917877322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:17:36.185643673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35760"
ts=2026-04-29T00:17:43.920508529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-29T00:17:49.25793703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:18:13.923442048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-29T00:18:36.498937349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:18:43.926236839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:18:50.293861363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55510"
ts=2026-04-29T00:19:06.192981546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33066"
ts=2026-04-29T00:19:13.928858357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=debug ts=2026-04-29T00:19:30.162578337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:19:43.931994079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:20:13.935093637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:20:21.572782906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:20:21.574299215Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:20:43.937860849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:20:50.301806725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60574"
ts=2026-04-29T00:21:01.204020534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39854"
ts=2026-04-29T00:21:13.941343465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:21:18.337571515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:21:31.206779368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59864"
ts=2026-04-29T00:21:36.204482672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37636"
ts=2026-04-29T00:21:43.94456807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:21:50.306133091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36798"
ts=2026-04-29T00:22:13.947505169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:22:18.151675763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:22:20.308908725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46528"
ts=2026-04-29T00:22:36.210221051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42586"
ts=2026-04-29T00:22:43.951007391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:22:56.257709791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:23:13.953928271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=error ts=2026-04-29T00:23:23.62457919Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:23:23.626306151Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:23:23.627918205Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:23:23.627941525Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:23:43.95720515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:23:49.870563886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:23:50.31585751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35714"
ts=2026-04-29T00:24:01.219289603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38954"
ts=2026-04-29T00:24:06.217814477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44944"
ts=2026-04-29T00:24:13.959783902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:24:36.22117761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58854"
level=debug ts=2026-04-29T00:24:42.7771619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:24:42.778750331Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:24:43.962380424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:25:13.965382417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:25:41.682731361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:25:43.968316307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:26:13.971829716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:26:20.324841932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56336"
level=debug ts=2026-04-29T00:26:36.365337212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:26:43.974905581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:27:01.233169762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34858"
ts=2026-04-29T00:27:06.231642065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56548"
ts=2026-04-29T00:27:13.977702954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:27:15.546218574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:27:36.234011128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35152"
ts=2026-04-29T00:27:43.980320275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:27:59.836575807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:28:01.238973701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39380"
ts=2026-04-29T00:28:13.983197723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:28:31.242038981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56718"
level=debug ts=2026-04-29T00:28:34.590697103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:28:34.592495975Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:28:43.986191904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:29:01.244811122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53744"
ts=2026-04-29T00:29:06.240248474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48908"
level=debug ts=2026-04-29T00:29:10.168715931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:29:13.989579318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:29:20.337002248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59342"
ts=2026-04-29T00:29:43.992978933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:30:06.245489898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59880"
level=debug ts=2026-04-29T00:30:08.874496736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:30:13.99653549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:30:20.342595844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40896"
ts=2026-04-29T00:30:31.251980572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55522"
ts=2026-04-29T00:30:43.999984931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
level=debug ts=2026-04-29T00:30:49.7683928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:31:06.250257433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52610"
ts=2026-04-29T00:31:14.002978609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:31:20.346485594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54676"
ts=2026-04-29T00:31:31.258149032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:53902"
level=debug ts=2026-04-29T00:31:42.293369372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:31:44.006024554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:32:14.009732342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:32:20.043310403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:32:20.045007533Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T00:32:36.256565635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59144"
ts=2026-04-29T00:32:44.012862443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6hg25-8bba4450 10.129.0.83:7946"
ts=2026-04-29T00:32:50.354552654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54508"
level=debug ts=2026-04-29T00:32:55.951603712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:33:14.015581594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
level=error ts=2026-04-29T00:33:23.624125217Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:33:23.625849075Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:33:23.62753607Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T00:33:23.627557194Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T00:33:41.985342493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:33:44.019386102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:33:50.358709723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45260"
ts=2026-04-29T00:34:06.262454449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47572"
ts=2026-04-29T00:34:14.022941656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:34:20.361931123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54742"
level=debug ts=2026-04-29T00:34:23.878639093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:34:31.27318858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:55334"
ts=2026-04-29T00:34:44.026474525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-21be26b7 10.128.0.73:7946"
ts=2026-04-29T00:35:01.276295631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59962"
level=debug ts=2026-04-29T00:35:01.996564674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T00:35:06.267854354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43006"
ts=2026-04-29T00:35:14.029835171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
ts=2026-04-29T00:35:36.270745782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39880"
ts=2026-04-29T00:35:44.033361479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-82pv4-15d34ffb 10.129.0.81:7946"
level=debug ts=2026-04-29T00:35:53.749172648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T00:35:53.750853822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"