level=info ts=2026-05-15T10:48:48.610569629Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-15T10:48:48.610601638Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-15T10:48:48.612466689Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-15T10:48:48.612787739Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-15T10:48:48.612863594Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.53 inf=eth0
level=debug ts=2026-05-15T10:48:48.613639588Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-15T10:48:48.613713739Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.53 inf=eth0
level=info ts=2026-05-15T10:48:48.613744184Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-213936e0
ts=2026-05-15T10:48:48.613806008Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-15T10:48:48.613816705Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.53 advertisePort=7946
level=debug ts=2026-05-15T10:48:48.613847635Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.53 advertisePort=7946
level=info ts=2026-05-15T10:48:48.613954581Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-15T10:48:48.619010598Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-15T10:48:48.619029856Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-15T10:48:48.619040223Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.087768ms
level=error ts=2026-05-15T10:48:48.61904861Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-15T10:48:48.619060782Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-15T10:48:48.621215841Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T10:48:48.621235145Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-15T10:48:48.629147744Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-15T10:48:48.629195022Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-15T10:48:48.629208411Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-15T10:48:48.62921882Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-15T10:48:48.629236886Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-15T10:48:48.629229096Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-15T10:48:48.629376796Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-15T10:48:48.629393693Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-15T10:48:48.629493006Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-15T10:48:48.629514135Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-15T10:48:48.629587554Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-15T10:48:48.629602977Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-15T10:48:48.629609333Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-15T10:48:48.629749392Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-15T10:48:48.62975819Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-15T10:48:48.629767066Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-15T10:48:49.630121789Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-15T10:48:49.740750461Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-15T10:48:49.740802354Z caller=loki.go:615 msg="Loki started" startup_time=1.193115437s
ts=2026-05-15T10:48:50.336440989Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T10:48:50.336469698Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:48:53.030591149Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T10:48:53.030615524Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-15T10:48:54.741421491Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-15T10:48:54.741481875Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-15T10:48:57.625560887Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T10:48:57.625586103Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-15T10:49:07.493590114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:49:10.243665113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.68:7946"
ts=2026-05-15T10:49:10.246919126Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.70:7946: dial tcp 10.129.0.70:7946: connect: connection refused"
ts=2026-05-15T10:49:10.248060913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.71:7946"
level=info ts=2026-05-15T10:49:10.249301302Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=21.630236167s
level=debug ts=2026-05-15T10:49:27.492346863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:49:37.660229744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:49:38.541166371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:45272"
level=debug ts=2026-05-15T10:49:49.605540483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:50:07.663766083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:50:14.707707002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51422"
level=debug ts=2026-05-15T10:50:18.102212481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:50:37.666773846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T10:50:38.442907059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-15T10:50:38.444786645Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T10:50:44.710678095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37792"
ts=2026-05-15T10:50:49.918365247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44136"
ts=2026-05-15T10:51:07.670366329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T10:51:21.366694788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:51:37.673382104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T10:51:38.551173634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36838"
ts=2026-05-15T10:51:49.923357986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50912"
ts=2026-05-15T10:52:07.676169718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:52:19.92572706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53802"
level=debug ts=2026-05-15T10:52:27.121347799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:52:37.678468609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:53:07.679574288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:53:14.723770827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47090"
level=debug ts=2026-05-15T10:53:23.584828204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:53:37.681086028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:53:44.727140158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53014"
ts=2026-05-15T10:53:49.932104409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58670"
ts=2026-05-15T10:54:07.68311329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T10:54:08.564083477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57240"
level=debug ts=2026-05-15T10:54:32.168990965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T10:54:37.686047815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:54:38.565744407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41474"
ts=2026-05-15T10:55:07.688925681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T10:55:14.73343109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43298"
ts=2026-05-15T10:55:19.938956842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48178"
level=debug ts=2026-05-15T10:55:23.979130728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-15T10:55:23.980872298Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T10:55:37.691978761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:55:49.941918305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56976"
ts=2026-05-15T10:56:07.694548293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T10:56:14.672922502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T10:56:37.69691278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T10:56:38.576187849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52200"
ts=2026-05-15T10:56:44.741390777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51640"
ts=2026-05-15T10:56:49.946298563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54060"
level=debug ts=2026-05-15T10:57:00.646656029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T10:57:07.69869306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:57:37.700241992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:57:38.580593922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43874"
ts=2026-05-15T10:57:49.950375029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53024"
level=debug ts=2026-05-15T10:57:55.629600396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T10:58:07.702572164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T10:58:37.705556165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T10:58:38.585889346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42108"
level=debug ts=2026-05-15T10:58:42.514233799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-15T10:58:54.741785118Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-15T10:58:54.747187018Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T10:58:54.749243802Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T10:58:54.75179113Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T10:58:54.751821056Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-15T10:58:54.751833643Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-15T10:59:07.707531797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:59:14.757319318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42786"
level=debug ts=2026-05-15T10:59:29.78957448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T10:59:29.791376064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T10:59:37.709174901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T10:59:38.589880187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:35932"
ts=2026-05-15T11:00:07.711689788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:00:08.591133931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:50584"
ts=2026-05-15T11:00:14.76366317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60000"
level=debug ts=2026-05-15T11:00:24.010138677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:00:37.713673223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:01:01.708738752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:01:07.71488607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:01:14.768098132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36528"
ts=2026-05-15T11:01:37.71764548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:01:38.598582392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58574"
level=debug ts=2026-05-15T11:01:43.347072135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:01:49.969682113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60426"
ts=2026-05-15T11:02:07.720116162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:02:19.972760647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45568"
level=debug ts=2026-05-15T11:02:23.679746861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:02:37.722476858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:02:44.774882555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33744"
level=debug ts=2026-05-15T11:02:58.230315097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:02:58.232086251Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:03:07.725392196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:03:37.727240034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:03:56.471063708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:04:07.729417695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:04:19.982512292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51808"
ts=2026-05-15T11:04:37.732285088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:04:44.784709692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52412"
level=debug ts=2026-05-15T11:04:51.280632787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:05:07.734985508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:05:08.617957676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33296"
ts=2026-05-15T11:05:14.787914667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60444"
ts=2026-05-15T11:05:37.738241394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:05:40.888425337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:05:49.990831479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40244"
ts=2026-05-15T11:06:07.741942534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:06:15.532593655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:06:19.993776361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57038"
ts=2026-05-15T11:06:37.745736216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:06:38.625608362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:35600"
ts=2026-05-15T11:06:49.997592403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54184"
level=debug ts=2026-05-15T11:06:54.581329562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:06:54.583245845Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:07:07.749167387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:07:14.799291685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33786"
level=debug ts=2026-05-15T11:07:30.820355295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:07:37.75248123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:07:38.629561373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54286"
ts=2026-05-15T11:08:07.756052209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:08:07.933125904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:08:37.758686913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=error ts=2026-05-15T11:08:54.754862358Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:08:54.756688731Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:08:54.75851926Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:08:54.758542535Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-15T11:08:55.811285017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:09:07.760878559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:09:08.636928999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53428"
ts=2026-05-15T11:09:14.808065243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40648"
ts=2026-05-15T11:09:20.009371451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43824"
level=debug ts=2026-05-15T11:09:33.160587437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:09:37.762926469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:09:50.012947761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51574"
ts=2026-05-15T11:10:07.765905865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:10:11.422521338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:10:11.424379151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:10:14.814608063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35382"
ts=2026-05-15T11:10:37.768524468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:10:38.64482996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41838"
ts=2026-05-15T11:10:44.817751711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55526"
ts=2026-05-15T11:10:50.020275372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37834"
level=debug ts=2026-05-15T11:10:55.686418771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:11:07.770617958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:11:20.023198377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36442"
ts=2026-05-15T11:11:37.776156424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:11:38.655913802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41388"
level=debug ts=2026-05-15T11:11:43.793449151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:11:44.822834573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50540"
ts=2026-05-15T11:12:07.779024232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:12:14.826314911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38256"
ts=2026-05-15T11:12:20.028158505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54380"
level=debug ts=2026-05-15T11:12:23.025284663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:12:37.782202735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:12:44.833566836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46798"
level=debug ts=2026-05-15T11:12:57.663378706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:13:07.785910503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:13:20.034474107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46418"
ts=2026-05-15T11:13:37.788574488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:13:38.667765054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:38394"
level=debug ts=2026-05-15T11:13:45.658193529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:13:45.66064101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:13:50.037896655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44902"
ts=2026-05-15T11:14:07.791066258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:14:14.84133546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56978"
level=debug ts=2026-05-15T11:14:30.339438086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:14:37.794220059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:14:44.847522291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59960"
ts=2026-05-15T11:15:07.796991768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:15:20.046336441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38636"
level=debug ts=2026-05-15T11:15:22.378455953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:15:37.799578476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:15:44.853814397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40340"
ts=2026-05-15T11:16:07.801969819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:16:08.520743693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:16:37.8043544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:16:44.859996455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35008"
level=debug ts=2026-05-15T11:16:58.473277623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:17:07.80648433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:17:37.809130263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:17:39.716358853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:17:39.718063405Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:17:44.864636224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59576"
ts=2026-05-15T11:18:07.810008963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:18:20.061445796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55374"
level=debug ts=2026-05-15T11:18:26.343675022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:18:37.812737766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=error ts=2026-05-15T11:18:54.7602346Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:18:54.7634953Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:18:54.766566985Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:18:54.766617607Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:19:07.815706538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:19:14.873805703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57784"
level=debug ts=2026-05-15T11:19:18.572502535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:19:37.818351996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:19:44.876584446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51222"
level=debug ts=2026-05-15T11:19:53.574494354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:20:07.821236845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:20:08.706617146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:34534"
ts=2026-05-15T11:20:14.879323983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47368"
level=debug ts=2026-05-15T11:20:30.321741242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:20:37.824337406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:20:44.882335479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49360"
ts=2026-05-15T11:21:07.826349228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:21:08.710923639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52694"
level=debug ts=2026-05-15T11:21:15.384784929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:21:15.386569686Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:21:37.827957513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:21:38.712567976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33742"
ts=2026-05-15T11:21:50.083932939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37426"
level=debug ts=2026-05-15T11:21:59.061675584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:22:07.829912371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:22:36.981429391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:22:37.832861622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:23:07.834693576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:23:14.894679614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36850"
level=debug ts=2026-05-15T11:23:17.825320764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:23:37.835772514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:23:50.09770166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55390"
level=debug ts=2026-05-15T11:24:03.058949912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:24:07.837741833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:24:37.840008768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:24:38.728982299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48062"
level=debug ts=2026-05-15T11:24:43.664185984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:24:43.666148077Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:25:07.842499676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:25:08.73066212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54388"
ts=2026-05-15T11:25:20.104632923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48864"
level=debug ts=2026-05-15T11:25:21.209335334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:25:37.844332015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:25:38.732137975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43936"
ts=2026-05-15T11:26:07.848752412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:26:14.907058455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40854"
level=debug ts=2026-05-15T11:26:17.425896486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:26:37.850804604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:26:44.910397378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37232"
level=debug ts=2026-05-15T11:26:59.697611534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:27:07.854088884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:27:14.913273726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60152"
ts=2026-05-15T11:27:37.856443726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:27:38.742123827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42818"
ts=2026-05-15T11:27:44.916910087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50550"
level=debug ts=2026-05-15T11:27:46.254819077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:28:07.859389597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:28:14.920444699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46534"
ts=2026-05-15T11:28:20.119806709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59198"
level=debug ts=2026-05-15T11:28:32.394440039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:28:32.396109237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:28:37.862246039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=error ts=2026-05-15T11:28:54.758323256Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:28:54.760300251Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:28:54.762318906Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:28:54.762343909Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:29:07.865342874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:29:08.750144234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57574"
level=debug ts=2026-05-15T11:29:09.183684024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:29:37.867528905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:29:44.925908018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34870"
level=debug ts=2026-05-15T11:30:01.301330143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:30:07.869929706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:30:37.872154282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:30:55.985227105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:31:07.874857222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:31:08.76021253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:44304"
ts=2026-05-15T11:31:37.877683978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:31:50.968703401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:32:07.8813217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:32:08.765205767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:55598"
ts=2026-05-15T11:32:14.935279387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51282"
ts=2026-05-15T11:32:37.885079564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:32:38.766448012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49280"
ts=2026-05-15T11:32:44.938712782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53610"
level=debug ts=2026-05-15T11:32:49.802526488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:32:49.804148759Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:33:07.887950555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:33:30.41377491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:33:37.89135468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:33:38.771477712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:38530"
ts=2026-05-15T11:33:44.944234607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55868"
ts=2026-05-15T11:34:07.893912292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:34:21.201443276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:34:37.897118892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:34:50.150832337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55830"
level=debug ts=2026-05-15T11:35:07.218705663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:35:07.900345117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:35:37.902688001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:35:38.782935622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57114"
level=debug ts=2026-05-15T11:35:42.304000791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:35:50.155200686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45896"
ts=2026-05-15T11:36:07.904551813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:36:14.958875784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40016"
level=debug ts=2026-05-15T11:36:16.423919251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:36:16.425984891Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:36:20.158151577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50364"
ts=2026-05-15T11:36:37.906127072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:37:07.908692139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:37:15.160151375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:37:37.910493945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:37:44.966545443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46002"
level=debug ts=2026-05-15T11:37:59.541885005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:38:07.913140054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:38:35.612918972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:38:37.916713387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:38:38.79871447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41516"
level=error ts=2026-05-15T11:38:54.75647982Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:38:54.758801093Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:38:54.760697835Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:38:54.760749216Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:39:07.920309291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:39:10.750354086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:39:20.173322613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:49170"
ts=2026-05-15T11:39:37.923157212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:39:50.334446166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:39:50.336342071Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:40:07.929548609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:40:14.97975876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:34858"
ts=2026-05-15T11:40:20.177464428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44658"
level=debug ts=2026-05-15T11:40:27.508700089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:40:37.932092976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:40:44.983331305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43106"
ts=2026-05-15T11:40:50.180658144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42876"
ts=2026-05-15T11:41:07.935580491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:41:20.183195987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47026"
level=debug ts=2026-05-15T11:41:24.818925835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:41:37.938624839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:41:44.988619431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50634"
level=debug ts=2026-05-15T11:42:05.435542648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:42:07.940767988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:42:20.189095847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58026"
ts=2026-05-15T11:42:37.94318875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:42:55.889765918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:43:07.946308151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:43:08.824809775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53968"
ts=2026-05-15T11:43:20.195345638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54004"
ts=2026-05-15T11:43:37.949931112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:43:42.796662001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:43:42.799034947Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:44:07.952097262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:44:24.663363088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:44:37.954835713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:44:38.832079981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58512"
ts=2026-05-15T11:45:07.95814171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:45:08.833266474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:45816"
ts=2026-05-15T11:45:15.006011214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41348"
ts=2026-05-15T11:45:20.205509445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60966"
level=debug ts=2026-05-15T11:45:24.083857198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:45:37.961284557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:45:50.208516284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52634"
ts=2026-05-15T11:46:07.964177448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:46:20.211950255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56206"
level=debug ts=2026-05-15T11:46:21.549898085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:46:37.967136156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:46:38.840682333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:55180"
ts=2026-05-15T11:46:45.013490992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46544"
ts=2026-05-15T11:46:50.215030381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52344"
ts=2026-05-15T11:47:07.970301512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:47:20.525475776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:47:37.973024004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:48:07.974717293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:48:08.84791013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54392"
level=debug ts=2026-05-15T11:48:18.326723114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:48:18.328390833Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:48:20.223149477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51116"
ts=2026-05-15T11:48:37.975564057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:48:45.024878254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49482"
ts=2026-05-15T11:48:50.225906262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51548"
level=error ts=2026-05-15T11:48:54.756856147Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:48:54.759180148Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:48:54.761543932Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:48:54.761576847Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:49:07.977615467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:49:15.747872197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:49:37.980751833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:50:07.983922553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:50:08.85891834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:55730"
level=debug ts=2026-05-15T11:50:14.091863879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:50:15.032777451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55942"
ts=2026-05-15T11:50:37.986676597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:50:38.860201491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42392"
level=debug ts=2026-05-15T11:51:00.32149314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:51:07.989603116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:51:15.037955369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52480"
level=debug ts=2026-05-15T11:51:37.597163595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:51:37.99246111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:51:45.040465876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49828"
ts=2026-05-15T11:52:07.995315864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:52:08.867849937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:45980"
level=debug ts=2026-05-15T11:52:19.781918453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:52:19.783721148Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:52:37.998346263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:53:06.59166127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:53:08.001363098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:53:08.871988941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41618"
ts=2026-05-15T11:53:20.246880226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40498"
ts=2026-05-15T11:53:38.004063755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:53:46.823125452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:53:50.250172371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55326"
ts=2026-05-15T11:54:08.006914894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T11:54:20.257409957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:38744"
level=debug ts=2026-05-15T11:54:24.613312087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:54:38.009406229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:54:45.056678831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41472"
ts=2026-05-15T11:54:50.260883271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42430"
ts=2026-05-15T11:55:08.011681023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T11:55:10.948466426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:55:38.013864664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:55:38.886847297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49466"
level=debug ts=2026-05-15T11:56:00.816664464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:56:00.818526508Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:56:08.015656778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:56:38.016635231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T11:56:51.929039099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:57:08.018310029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:57:20.27090876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34588"
ts=2026-05-15T11:57:38.02011885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:57:38.898073554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39946"
ts=2026-05-15T11:57:45.069018621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50998"
level=debug ts=2026-05-15T11:57:48.690529072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:58:08.022357709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T11:58:33.942867002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:58:38.025741017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T11:58:38.907665473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37968"
ts=2026-05-15T11:58:45.075472821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43466"
level=error ts=2026-05-15T11:58:54.756266538Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:58:54.758072132Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:58:54.759750756Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:58:54.759769955Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:59:08.027911356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:59:15.078954011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55004"
ts=2026-05-15T11:59:20.277969026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60800"
level=debug ts=2026-05-15T11:59:30.466737864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:59:38.028941029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T11:59:38.911756513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:46330"
ts=2026-05-15T11:59:45.082411342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37724"
ts=2026-05-15T12:00:08.031407192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:00:08.913339549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:32870"
level=debug ts=2026-05-15T12:00:11.998808284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:00:12.000702007Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:00:20.284544184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40416"
ts=2026-05-15T12:00:38.034411232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:00:45.088370054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54548"
ts=2026-05-15T12:00:50.287628215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34282"
ts=2026-05-15T12:01:08.036425477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T12:01:08.370804514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:01:15.091781138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51306"
ts=2026-05-15T12:01:38.038383442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:01:50.292819264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53454"
ts=2026-05-15T12:02:08.041711835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T12:02:08.238456735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:02:08.924182973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36326"
ts=2026-05-15T12:02:38.044634576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:02:44.901095066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:03:08.046789916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:03:08.929476871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:60604"
ts=2026-05-15T12:03:20.300693713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59924"
level=debug ts=2026-05-15T12:03:28.162857004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:03:38.049080519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:03:50.303693169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45478"
ts=2026-05-15T12:04:08.052485106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:04:15.107593709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47008"
level=debug ts=2026-05-15T12:04:16.39224847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:04:16.394016748Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:04:38.054984191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:04:45.111093756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52580"
ts=2026-05-15T12:04:50.309952556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42170"
level=debug ts=2026-05-15T12:05:05.926946976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:05:08.056417265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:05:38.05883754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T12:05:41.216722825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:06:08.062199123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:06:08.946601251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54956"
ts=2026-05-15T12:06:15.118046904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39214"
level=debug ts=2026-05-15T12:06:35.415477113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:06:38.065062203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:06:50.316979333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45104"
ts=2026-05-15T12:07:08.068112579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:07:20.319541061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54276"
level=debug ts=2026-05-15T12:07:29.484006944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:07:38.070564647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:07:45.125731647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58748"
ts=2026-05-15T12:08:08.072978952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:08:12.756154151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:08:12.758057655Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:08:15.128863289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36216"
ts=2026-05-15T12:08:38.07592917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=error ts=2026-05-15T12:08:54.755904563Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:08:54.757680003Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:08:54.759589371Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:08:54.759618138Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-15T12:09:02.318043545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:09:08.078555161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:09:38.080928043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:10:02.099820252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:10:08.082909245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T12:10:36.093229765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:10:38.085510338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:10:38.973545021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37772"
ts=2026-05-15T12:11:08.088159301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:11:08.975402613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36206"
level=debug ts=2026-05-15T12:11:22.671432809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:11:38.09000295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:11:45.149644985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51436"
ts=2026-05-15T12:12:08.093003065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T12:12:21.256006113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:12:21.257663708Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:12:38.095953626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
level=debug ts=2026-05-15T12:13:05.356599909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:13:08.09843601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:13:15.157134994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59006"
ts=2026-05-15T12:13:38.100974336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:13:43.244199452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:14:08.103939006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:14:30.661089931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:14:38.106619203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:14:45.16359367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49404"
ts=2026-05-15T12:15:08.109316454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:15:08.998748318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52742"
ts=2026-05-15T12:15:15.166834161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43650"
level=debug ts=2026-05-15T12:15:23.308189898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:15:38.112437586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:15:39.000007264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54166"
ts=2026-05-15T12:15:50.35456414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60550"
ts=2026-05-15T12:16:08.115150654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:16:09.001909784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:60128"
ts=2026-05-15T12:16:15.171925305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54510"
level=debug ts=2026-05-15T12:16:20.646165106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:16:20.647889596Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:16:38.118046444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:16:39.003456047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58184"
ts=2026-05-15T12:16:50.358773182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46954"
level=debug ts=2026-05-15T12:16:58.173143604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:17:08.121063953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:17:15.176910108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41882"
ts=2026-05-15T12:17:20.361810021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42004"
ts=2026-05-15T12:17:38.123975675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:17:39.472399757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:18:08.126997987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:18:38.128810214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T12:18:38.856231684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:18:45.184321181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42244"
ts=2026-05-15T12:18:50.366996997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53200"
level=error ts=2026-05-15T12:18:54.756605141Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:18:54.758592218Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:18:54.760575908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:18:54.760609351Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:19:08.131117254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:19:20.37028662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60810"
level=debug ts=2026-05-15T12:19:33.28414647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:19:38.133917875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:19:45.189085437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37500"
ts=2026-05-15T12:20:08.136461499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:20:09.376221022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:20:09.377813088Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:20:38.139720739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:20:39.027643455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:40402"
level=debug ts=2026-05-15T12:20:44.616919636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:20:50.378983649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59398"
ts=2026-05-15T12:21:08.141665498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:21:15.198587046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:35610"
ts=2026-05-15T12:21:38.142701227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T12:21:45.414609904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:22:08.144549395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:22:20.387906977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52330"
ts=2026-05-15T12:22:38.146449714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:22:42.26231476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:22:45.206660679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52938"
ts=2026-05-15T12:23:08.148710996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
level=debug ts=2026-05-15T12:23:29.004817117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:23:38.155781437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:24:08.158298353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:24:09.047324188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:50208"
level=debug ts=2026-05-15T12:24:23.26946261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:24:23.271272945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:24:38.159795867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2856287a 10.128.0.54:7946"
ts=2026-05-15T12:24:39.049116848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52732"
ts=2026-05-15T12:24:50.399666671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54528"
ts=2026-05-15T12:25:08.161787665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:25:20.402448693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46856"
level=debug ts=2026-05-15T12:25:24.582647936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:25:38.165222449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:26:06.31130116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:26:08.168549717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:26:09.056490192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53812"
ts=2026-05-15T12:26:38.172018396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
level=debug ts=2026-05-15T12:26:48.727127396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:27:08.175214948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:27:38.178287996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:27:39.064049564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39842"
level=debug ts=2026-05-15T12:27:45.130914483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:28:08.181147492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hhk7q-d2a47839 10.129.0.68:7946"
ts=2026-05-15T12:28:20.419671475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41056"
level=debug ts=2026-05-15T12:28:23.067857982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:28:23.069420369Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:28:38.183820082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:28:50.422132775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33850"
level=error ts=2026-05-15T12:28:54.756387911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:28:54.758499301Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:28:54.760605519Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:28:54.760633218Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-15T12:29:00.55566436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:29:08.186509612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"
ts=2026-05-15T12:29:09.071178646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33992"
ts=2026-05-15T12:29:15.235589027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53888"
ts=2026-05-15T12:29:38.189640309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-jr8j6-7b7fa67d 10.129.0.71:7946"