level=info ts=2026-05-07T14:53:46.803409968Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T14:53:46.803450363Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T14:53:46.806605677Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T14:53:46.807043136Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:53:46.807159035Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.69 inf=eth0
level=debug ts=2026-05-07T14:53:46.807848828Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:53:46.807929532Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.69 inf=eth0
level=info ts=2026-05-07T14:53:46.807980194Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-77d03d88
ts=2026-05-07T14:53:46.808059474Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T14:53:46.808085663Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=debug ts=2026-05-07T14:53:46.808104656Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=info ts=2026-05-07T14:53:46.808296328Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T14:53:46.817312675Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T14:53:46.817335583Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:53:46.817347881Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=9.055003ms
level=error ts=2026-05-07T14:53:46.817356636Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T14:53:46.817370541Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T14:53:46.821248573Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:53:46.821265741Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:53:46.826106493Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-07T14:53:46.826117554Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T14:53:46.826114631Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-07T14:53:46.826151335Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-07T14:53:46.826171906Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-07T14:53:46.826207127Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T14:53:46.826740634Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-07T14:53:46.826751518Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-07T14:53:46.826755113Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-07T14:53:46.826759576Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T14:53:46.826908134Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T14:53:46.826907697Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T14:53:46.826930197Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T14:53:46.827109411Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T14:53:46.993410846Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T14:53:46.993494803Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-07T14:53:47.993993379Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:53:47.99401482Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T14:53:47.994044444Z caller=loki.go:615 msg="Loki started" startup_time=1.260740238s
ts=2026-05-07T14:53:48.8152382Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:53:48.815263319Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:53:50.953552673Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:53:50.953580393Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:53:52.994514667Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T14:53:52.994578745Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T14:53:57.055971084Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:53:57.055993714Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T14:54:01.316192878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:54:06.768801839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51958"
ts=2026-05-07T14:54:09.678839958Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
ts=2026-05-07T14:54:09.679449226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.75:7946"
ts=2026-05-07T14:54:09.680823924Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-05-07T14:54:09.680908627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.69:7946"
ts=2026-05-07T14:54:09.680934047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34042"
level=info ts=2026-05-07T14:54:09.681174568Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.863799378s
ts=2026-05-07T14:54:11.110344399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42882"
ts=2026-05-07T14:54:14.033405812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:46716"
level=debug ts=2026-05-07T14:54:19.472460026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:54:22.405964683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T14:54:28.838102882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39916"
level=debug ts=2026-05-07T14:54:38.813389921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:54:40.265736025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55964"
ts=2026-05-07T14:54:43.787365784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:45728"
ts=2026-05-07T14:54:52.40815768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T14:55:01.600564498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:55:22.409725605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T14:55:28.841453552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39002"
level=debug ts=2026-05-07T14:55:35.595440617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T14:55:35.597080488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T14:55:40.269448096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57408"
ts=2026-05-07T14:55:52.411378921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T14:55:58.842700145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50734"
ts=2026-05-07T14:56:13.792321594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:50870"
level=debug ts=2026-05-07T14:56:14.073802493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:56:22.412851636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T14:56:28.843994892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51604"
ts=2026-05-07T14:56:40.27322124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59626"
ts=2026-05-07T14:56:43.7948355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37994"
ts=2026-05-07T14:56:52.414488526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T14:57:10.275653148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42226"
ts=2026-05-07T14:57:22.415821224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T14:57:24.049970034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:57:28.847150497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51714"
ts=2026-05-07T14:57:52.417595285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T14:57:58.848908833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38902"
ts=2026-05-07T14:58:10.280044759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44826"
ts=2026-05-07T14:58:13.800093494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:34580"
ts=2026-05-07T14:58:22.420892589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T14:58:24.809761286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:58:40.282623215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35284"
ts=2026-05-07T14:58:52.423089997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T14:59:22.425493959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T14:59:33.403094005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T14:59:40.286338025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49348"
ts=2026-05-07T14:59:52.427700474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T14:59:58.855492539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58092"
ts=2026-05-07T15:00:22.429817661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:00:25.9085554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:00:25.910237342Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:00:52.431857999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:00:58.858295898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33172"
level=debug ts=2026-05-07T15:01:04.006048203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:01:22.434093165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:01:28.859996156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58076"
ts=2026-05-07T15:01:52.436614558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:01:56.264333118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:01:58.861476397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37808"
ts=2026-05-07T15:02:10.297324763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:60572"
ts=2026-05-07T15:02:22.438422649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:02:28.862895239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55522"
ts=2026-05-07T15:02:52.440744553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:02:54.254499634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:02:58.864316922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41540"
ts=2026-05-07T15:03:22.442455669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:03:28.865620157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60074"
level=debug ts=2026-05-07T15:03:35.25894125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:03:43.81828845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43224"
ts=2026-05-07T15:03:52.443935375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=info ts=2026-05-07T15:03:52.99555717Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T15:03:52.999440456Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:03:53.00140583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:03:53.003225802Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:03:53.003260272Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:03:53.003274295Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T15:03:58.866896192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47914"
level=debug ts=2026-05-07T15:04:22.405318047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:04:22.406880646Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:04:22.445044611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:04:28.868284513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56100"
ts=2026-05-07T15:04:52.446194047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:05:10.305620106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35872"
ts=2026-05-07T15:05:13.824526137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:49742"
level=debug ts=2026-05-07T15:05:16.816372067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:05:22.448390201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:05:40.307993927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56130"
ts=2026-05-07T15:05:52.450808699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:05:52.822509522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:06:10.309768663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55280"
ts=2026-05-07T15:06:13.828366326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:52772"
ts=2026-05-07T15:06:22.453525268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:06:39.335168368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:06:43.830123803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:60472"
ts=2026-05-07T15:06:52.455649602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:06:58.877626364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51722"
ts=2026-05-07T15:07:10.313174597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:60446"
ts=2026-05-07T15:07:13.832615864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37596"
level=debug ts=2026-05-07T15:07:14.921981679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:07:22.457883001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:07:40.315267683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45548"
level=debug ts=2026-05-07T15:07:52.194513201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:07:52.196458291Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:07:52.459856928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:07:58.881321082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34298"
ts=2026-05-07T15:08:13.836737645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59452"
ts=2026-05-07T15:08:22.461652872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:08:28.814835657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:08:52.463021975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:09:09.816128883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:09:10.321376237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:46546"
ts=2026-05-07T15:09:22.464936392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:09:40.323276516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35300"
ts=2026-05-07T15:09:52.467340013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:10:10.325489409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56438"
level=debug ts=2026-05-07T15:10:10.372782822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:13.843932954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:36236"
ts=2026-05-07T15:10:22.469464813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:10:28.897575139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38708"
ts=2026-05-07T15:10:43.846411261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:60336"
level=debug ts=2026-05-07T15:10:45.612945375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:10:52.471228628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:10:58.905643841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33040"
ts=2026-05-07T15:11:22.473322268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:11:23.969832905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:11:23.972219238Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:11:52.475693125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:11:58.909306225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44174"
level=debug ts=2026-05-07T15:12:07.256140323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:12:13.852162402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:55938"
ts=2026-05-07T15:12:22.477717601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:12:52.479697838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:13:03.243099096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:13:10.334933919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:53488"
ts=2026-05-07T15:13:13.855921902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59236"
ts=2026-05-07T15:13:22.481812767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:13:52.483732289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=error ts=2026-05-07T15:13:53.010331Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:13:53.014562683Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:13:53.017156771Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:13:53.01719003Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:13:58.919739653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56782"
level=debug ts=2026-05-07T15:14:02.521531952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:14:22.485474316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:14:43.861168402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:53692"
level=debug ts=2026-05-07T15:14:49.016812556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:14:52.487638931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:14:58.923882426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42142"
ts=2026-05-07T15:15:10.344590146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42846"
ts=2026-05-07T15:15:22.490054438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:15:26.32926031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:15:26.331091357Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:15:28.925332615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41844"
ts=2026-05-07T15:15:52.495749932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:16:10.347454119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:41142"
ts=2026-05-07T15:16:22.501704091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:16:24.70864693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:16:52.50443372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:16:58.932131769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34420"
level=debug ts=2026-05-07T15:17:00.7366972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:17:22.507194326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:17:40.353206749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54060"
level=debug ts=2026-05-07T15:17:52.199665921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:17:52.509181507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:17:58.93588415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54166"
ts=2026-05-07T15:18:13.874675579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:36976"
ts=2026-05-07T15:18:22.51140784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:18:28.937085208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35548"
level=debug ts=2026-05-07T15:18:39.469240306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:18:40.356743139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37464"
ts=2026-05-07T15:18:43.876712464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:52290"
ts=2026-05-07T15:18:52.513349056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:19:22.515330617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:19:26.617147984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:19:26.618747686Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:19:28.940323105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45218"
ts=2026-05-07T15:19:40.36046138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54230"
ts=2026-05-07T15:19:52.524787776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:19:58.941717802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32874"
level=debug ts=2026-05-07T15:20:12.82980233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:20:13.882781281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:33262"
ts=2026-05-07T15:20:22.52743168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:20:28.943512819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33204"
ts=2026-05-07T15:20:40.363661078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48062"
ts=2026-05-07T15:20:52.529769098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:20:58.945318814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54182"
level=debug ts=2026-05-07T15:21:05.153065503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:21:22.532564225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:21:40.366505309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:47172"
ts=2026-05-07T15:21:43.887999655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43682"
ts=2026-05-07T15:21:52.534550213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:21:58.948977486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32990"
level=debug ts=2026-05-07T15:22:03.374367006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:22:22.536901554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:22:41.654659664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:22:52.538756558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:23:15.888263674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:23:15.889904396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:23:22.540637413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:23:28.954961998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47274"
ts=2026-05-07T15:23:52.54302091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=error ts=2026-05-07T15:23:53.006340504Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:23:53.008350027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:23:53.010266126Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:23:53.010292552Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:23:55.473832035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:24:22.544693687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:24:40.376519548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:53800"
ts=2026-05-07T15:24:43.898851121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:55218"
level=debug ts=2026-05-07T15:24:46.985265318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:24:52.546089854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:25:13.900572266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:54410"
ts=2026-05-07T15:25:22.548198245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:25:32.061952715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:25:43.902995189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42042"
ts=2026-05-07T15:25:52.549765614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:26:10.383208525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59140"
ts=2026-05-07T15:26:22.55122574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:26:23.765947704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:26:28.968345552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51210"
ts=2026-05-07T15:26:43.906083097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:47292"
ts=2026-05-07T15:26:52.553613456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:27:01.163184097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:27:01.165367817Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:27:22.555784405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:27:40.138321909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:27:43.90994776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:46656"
ts=2026-05-07T15:27:52.558406317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:27:58.973773485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35718"
ts=2026-05-07T15:28:10.392451598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33030"
level=debug ts=2026-05-07T15:28:20.009336692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:28:22.560678224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:28:40.394489407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55658"
ts=2026-05-07T15:28:52.562569301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:28:54.747710439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:29:13.916524254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:34630"
ts=2026-05-07T15:29:22.563957231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:29:45.900523517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:29:52.566613359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:29:58.981083583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43042"
ts=2026-05-07T15:30:22.56885845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:30:40.401651278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33072"
level=debug ts=2026-05-07T15:30:47.468877854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:30:47.470513395Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:30:52.570882257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:31:10.404185355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:52800"
ts=2026-05-07T15:31:13.924557941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:53456"
ts=2026-05-07T15:31:22.57281844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:31:28.988066743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44532"
ts=2026-05-07T15:31:43.927110009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:44154"
level=debug ts=2026-05-07T15:31:49.301354786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:31:52.575153322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:32:22.577768835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T15:32:26.477251039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:32:52.579771062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:33:03.450273186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:33:22.581657094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:33:40.414099085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55256"
level=debug ts=2026-05-07T15:33:43.417139807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:33:52.583557181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=error ts=2026-05-07T15:33:53.006926358Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:33:53.008976708Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:33:53.010921539Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:33:53.010950159Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:34:21.397026571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:34:21.398699968Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:34:22.585717105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:34:28.999009292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44554"
ts=2026-05-07T15:34:52.586853533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:35:10.419290535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57630"
ts=2026-05-07T15:35:22.589066058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:35:22.956663509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:35:43.942221747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:54244"
ts=2026-05-07T15:35:52.590958215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:36:07.455812811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:36:22.593455798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:36:52.595634353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:36:54.623496614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:37:10.426663666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:50496"
ts=2026-05-07T15:37:22.597724892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:37:29.011139729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60980"
level=debug ts=2026-05-07T15:37:52.302804176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:37:52.598993605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:38:13.951404298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:39268"
ts=2026-05-07T15:38:22.605872373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:38:52.609656686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:38:53.935764454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:38:53.937399641Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:38:59.017123225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45422"
ts=2026-05-07T15:39:13.956203321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59612"
ts=2026-05-07T15:39:22.618697606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:39:29.02068992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44834"
ts=2026-05-07T15:39:43.958464407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:38664"
level=debug ts=2026-05-07T15:39:45.222667758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:39:52.620904641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:40:10.436781547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33056"
ts=2026-05-07T15:40:22.622488234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:40:29.024861953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38160"
ts=2026-05-07T15:40:40.439309118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:41340"
level=debug ts=2026-05-07T15:40:42.240647162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:40:52.62499203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:40:59.026622812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53262"
level=debug ts=2026-05-07T15:41:22.036693135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:41:22.626296834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:41:52.628432518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:42:04.901139017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:42:13.969075751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43906"
ts=2026-05-07T15:42:22.630239323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:42:29.032059889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34022"
ts=2026-05-07T15:42:52.632642358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:43:04.856201906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:43:04.858069478Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:43:10.448237131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51090"
ts=2026-05-07T15:43:22.635083528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:43:52.636289597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=error ts=2026-05-07T15:43:53.00788172Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:43:53.009830963Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:43:53.011822654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:43:53.011844604Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:44:06.046664827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:44:22.637918312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:44:29.039274801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34210"
ts=2026-05-07T15:44:52.639854402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:44:54.610023932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:22.641414753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:45:29.043101992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47730"
level=debug ts=2026-05-07T15:45:34.533900467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:45:40.459414962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:47026"
ts=2026-05-07T15:45:52.642775369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:46:13.984330665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48180"
level=debug ts=2026-05-07T15:46:19.410698864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:46:22.644389526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:46:52.646904099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:46:59.049309414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60612"
level=debug ts=2026-05-07T15:47:11.466027654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:47:11.467789208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:47:22.64906739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:47:43.989955171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41324"
ts=2026-05-07T15:47:52.650419272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:47:59.052768637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60930"
level=debug ts=2026-05-07T15:48:07.086816105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:48:13.992499635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43562"
ts=2026-05-07T15:48:22.651606369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:48:29.054307233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41684"
ts=2026-05-07T15:48:40.46875576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45854"
ts=2026-05-07T15:48:43.994437467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:38870"
level=debug ts=2026-05-07T15:48:49.85528749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:48:52.653580917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:49:22.656093001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:49:29.05721719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47890"
level=debug ts=2026-05-07T15:49:48.122070603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:49:52.65736155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:49:59.058415166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37110"
ts=2026-05-07T15:50:10.472500745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48502"
ts=2026-05-07T15:50:22.658625635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:50:31.568824005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:50:40.474139249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37448"
ts=2026-05-07T15:50:44.004119091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37224"
ts=2026-05-07T15:50:52.660391123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:51:08.277949124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:51:08.279505719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:51:10.476622819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42976"
ts=2026-05-07T15:51:22.662321657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:51:40.479008702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56760"
ts=2026-05-07T15:51:44.008898698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:51710"
level=debug ts=2026-05-07T15:51:49.91791631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:51:52.663862431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:52:22.665152982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:52:29.066745532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50936"
ts=2026-05-07T15:52:40.482145427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42188"
ts=2026-05-07T15:52:44.014023534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41818"
level=debug ts=2026-05-07T15:52:49.891296798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:52:52.666696244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:52:59.068731175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53322"
ts=2026-05-07T15:53:10.484407366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55640"
ts=2026-05-07T15:53:22.669317978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:53:48.802864981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:53:52.671335172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=error ts=2026-05-07T15:53:53.00692385Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:53:53.008984068Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:53:53.011021378Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T15:53:53.011043734Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:53:59.072160575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43796"
ts=2026-05-07T15:54:22.673780688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:54:29.073462105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55372"
level=debug ts=2026-05-07T15:54:41.869643681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:54:52.676449654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:55:10.490072803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:46254"
ts=2026-05-07T15:55:22.678588506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:55:33.139185702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:55:33.140892552Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:55:52.680431668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T15:56:10.493514763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55384"
level=debug ts=2026-05-07T15:56:12.606340451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:56:14.026417565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59370"
ts=2026-05-07T15:56:22.682634225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:56:40.495676345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55686"
ts=2026-05-07T15:56:44.028699684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48900"
ts=2026-05-07T15:56:52.684624063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:56:59.085062445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49712"
level=debug ts=2026-05-07T15:57:14.242271156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:57:22.686234201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T15:57:29.086159494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36838"
ts=2026-05-07T15:57:52.688164224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T15:57:55.537836013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:58:22.689396972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:58:29.089311764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48396"
level=debug ts=2026-05-07T15:58:37.355498926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:58:52.690804007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T15:59:10.504614365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36684"
ts=2026-05-07T15:59:14.037402261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:46756"
ts=2026-05-07T15:59:22.692402623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T15:59:31.7597515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:59:31.761288806Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T15:59:52.693835098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T16:00:08.68760893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:00:14.042141475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37096"
ts=2026-05-07T16:00:22.695890432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:00:40.511216212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49126"
level=debug ts=2026-05-07T16:00:45.658723725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:00:52.698421234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:00:59.097915897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45348"
ts=2026-05-07T16:01:14.046470629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42756"
ts=2026-05-07T16:01:22.700261362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:01:41.521668411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:01:52.702114761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:02:10.516169785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54620"
ts=2026-05-07T16:02:22.703426221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:02:36.130569569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:02:44.053938857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:60712"
ts=2026-05-07T16:02:52.705092077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:03:10.520848388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49800"
level=debug ts=2026-05-07T16:03:19.031121137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:03:19.032734288Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:03:22.707113212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:03:40.522918327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43018"
ts=2026-05-07T16:03:52.708864129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=error ts=2026-05-07T16:03:53.008189539Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:03:53.010159322Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:03:53.012189321Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:03:53.012211959Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T16:04:07.748213736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:04:10.525377284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37766"
ts=2026-05-07T16:04:14.059511686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:56222"
ts=2026-05-07T16:04:22.711151217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:04:29.113397762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43322"
ts=2026-05-07T16:04:40.527112032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36852"
ts=2026-05-07T16:04:44.062080117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:33740"
ts=2026-05-07T16:04:52.71339962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:05:05.619715921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:05:22.715008057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:05:40.530377212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:40052"
ts=2026-05-07T16:05:52.717591362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:06:00.770208986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:06:22.720133269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:06:29.120719216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37062"
ts=2026-05-07T16:06:52.72239306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:06:57.755070204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:06:59.121853703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44380"
ts=2026-05-07T16:07:10.535111626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45634"
ts=2026-05-07T16:07:22.724524794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:07:52.725899653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:07:55.764426408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:07:55.766113421Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:07:59.124836524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40348"
ts=2026-05-07T16:08:22.727203115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:08:39.890586874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:08:52.728890767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:08:59.128923435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46520"
ts=2026-05-07T16:09:22.731026314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:09:29.130724197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35474"
level=debug ts=2026-05-07T16:09:40.261893573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:09:44.082829131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:60288"
ts=2026-05-07T16:09:52.733444242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:10:10.545458133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57014"
level=debug ts=2026-05-07T16:10:19.35106634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:10:22.73544875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:10:29.135357185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48922"
ts=2026-05-07T16:10:40.54740584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36798"
ts=2026-05-07T16:10:44.086447412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:49238"
ts=2026-05-07T16:10:52.737026092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:11:07.912993442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:11:14.088767947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:33818"
ts=2026-05-07T16:11:22.739004937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:11:44.090800295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41538"
level=debug ts=2026-05-07T16:11:45.168995347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:11:45.170492822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:11:52.740350229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:12:14.092599659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:52000"
ts=2026-05-07T16:12:22.742008832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:12:38.278326975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:12:52.743621183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:12:59.146192291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57430"
ts=2026-05-07T16:13:14.097409366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:54692"
level=debug ts=2026-05-07T16:13:16.544375626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:13:22.745675167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:13:40.556491083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59998"
ts=2026-05-07T16:13:52.747735257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=error ts=2026-05-07T16:13:53.010083617Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:13:53.015364646Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:13:53.020060181Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:13:53.020082116Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T16:13:53.792315396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:13:59.154278111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38834"
ts=2026-05-07T16:14:22.749494515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:14:29.15515194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48976"
level=debug ts=2026-05-07T16:14:42.405749072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:14:44.10397192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:52738"
ts=2026-05-07T16:14:52.751510809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:15:22.753366035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:15:24.240602842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:15:24.242134754Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:15:52.754581345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:16:07.394739147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:16:22.755863896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:16:29.16274762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55400"
ts=2026-05-07T16:16:40.568513053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:42250"
ts=2026-05-07T16:16:44.11072162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42684"
ts=2026-05-07T16:16:52.757291537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:16:57.1759019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:17:22.758866386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:17:29.16660755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34172"
ts=2026-05-07T16:17:52.760728406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:17:56.011408752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:17:59.167869024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35988"
ts=2026-05-07T16:18:22.762847008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T16:18:30.987336128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:18:52.764888602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:18:59.172090134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43490"
ts=2026-05-07T16:19:22.766481346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:19:26.877875172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:19:26.879362605Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:19:29.173748787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41052"
ts=2026-05-07T16:19:52.767994626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:20:12.681131425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:20:22.769869159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:20:44.1247703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:46250"
ts=2026-05-07T16:20:52.772456923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:20:59.179812394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57286"
level=debug ts=2026-05-07T16:21:08.378645566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:21:22.77415677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:21:29.18158095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52390"
ts=2026-05-07T16:21:40.585377059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56846"
ts=2026-05-07T16:21:52.775277824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:21:59.183478223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45644"
level=debug ts=2026-05-07T16:22:05.561267333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:22:10.58780062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34114"
ts=2026-05-07T16:22:22.777512666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:22:29.184888974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56846"
ts=2026-05-07T16:22:52.780070236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T16:22:54.143495135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:23:22.781836038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:23:40.738634863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:23:40.740605295Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:23:52.78382666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=error ts=2026-05-07T16:23:53.007601049Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:23:53.009597665Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:23:53.01137946Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:23:53.0114014Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:24:10.595730591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35910"
ts=2026-05-07T16:24:22.785170708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:24:36.978578044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:24:40.59755796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54182"
ts=2026-05-07T16:24:44.135449334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:53102"
ts=2026-05-07T16:24:52.787055828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:25:14.137229674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:40878"
level=debug ts=2026-05-07T16:25:20.311444474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:25:22.789264498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:25:29.197743964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57306"
ts=2026-05-07T16:25:40.601857909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37776"
ts=2026-05-07T16:25:44.139184438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42782"
ts=2026-05-07T16:25:52.791934806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T16:26:21.764467551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:26:22.793259365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:26:44.143689886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:57660"
ts=2026-05-07T16:26:52.795154815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:26:59.202402147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37732"
ts=2026-05-07T16:27:10.608746421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43326"
ts=2026-05-07T16:27:14.145575409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:33706"
level=debug ts=2026-05-07T16:27:14.556168277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:27:22.796815787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:27:40.611298364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48018"
ts=2026-05-07T16:27:52.798844219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=debug ts=2026-05-07T16:27:55.311616974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:27:55.313520541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:28:22.800983887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:28:29.207643043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43688"
level=debug ts=2026-05-07T16:28:38.128830647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:28:52.802715828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:29:10.622468648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35050"
level=debug ts=2026-05-07T16:29:15.542609984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:29:22.804796591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:29:52.806990803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
level=debug ts=2026-05-07T16:30:09.274701233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:30:22.808304275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:30:29.215592556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38692"
ts=2026-05-07T16:30:44.15817862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42284"
ts=2026-05-07T16:30:52.810971536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:31:04.145664118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:31:10.63147547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35628"
ts=2026-05-07T16:31:22.813150905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:31:29.21879739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47642"
level=debug ts=2026-05-07T16:31:47.359445442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:31:47.360989022Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T16:31:52.815864491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:32:22.818420348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
level=debug ts=2026-05-07T16:32:27.601064737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:32:52.820634174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-7968n-ab9b9d51 10.128.0.75:7946"
ts=2026-05-07T16:32:59.224648857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42450"
level=debug ts=2026-05-07T16:33:19.022619649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:33:22.82273435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wwxk8-a994b565 10.129.0.67:7946"
ts=2026-05-07T16:33:52.824207832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
level=error ts=2026-05-07T16:33:53.009377118Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:33:53.011569083Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:33:53.013368032Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T16:33:53.013389197Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T16:34:16.296843903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T16:34:22.826517333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4f5ef1e8 10.128.0.80:7946"
ts=2026-05-07T16:34:29.230195893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50940"
ts=2026-05-07T16:34:44.174338797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:53962"