level=info ts=2026-05-11T11:38:11.711244399Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-11T11:38:11.711672299Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-11T11:38:11.712951474Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-11T11:38:11.713393125Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:38:11.713502831Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.89 inf=eth0
level=debug ts=2026-05-11T11:38:11.715199823Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:38:11.715310857Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.89 inf=eth0
level=info ts=2026-05-11T11:38:11.715458834Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-60884486
ts=2026-05-11T11:38:11.715546447Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-11T11:38:11.715582065Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.89 advertisePort=7946
level=debug ts=2026-05-11T11:38:11.715605601Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.89 advertisePort=7946
level=info ts=2026-05-11T11:38:11.715803529Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-11T11:38:11.724231037Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:38:11.724256433Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:38:11.724271968Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=8.471505ms
level=error ts=2026-05-11T11:38:11.724283308Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-11T11:38:11.724298781Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-11T11:38:11.726934013Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:11.726958809Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:38:11.734307446Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-11T11:38:11.734278795Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-11T11:38:11.734340341Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-11T11:38:11.734329801Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-11T11:38:11.734345488Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-11T11:38:11.734470153Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-11T11:38:11.734596357Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-11T11:38:11.73460659Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-11T11:38:11.73475071Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-11T11:38:11.73476193Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-11T11:38:11.734877315Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-11T11:38:11.734915082Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-11T11:38:11.735211252Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-11T11:38:11.735225325Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-11T11:38:11.735241198Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-11T11:38:12.736067922Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-11T11:38:12.736096888Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-11T11:38:12.736120188Z caller=loki.go:615 msg="Loki started" startup_time=1.099528598s
ts=2026-05-11T11:38:12.918260439Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:12.918284216Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:38:16.102419398Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:16.102445761Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T11:38:17.736838284Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-11T11:38:17.736891061Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-11T11:38:23.094254339Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:23.094282117Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:38:27.729542893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:38:33.391400331Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-05-11T11:38:33.391531343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.89:7946"
ts=2026-05-11T11:38:33.391597919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35672"
level=info ts=2026-05-11T11:38:33.391849124Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.667543663s
ts=2026-05-11T11:38:34.162433399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:32920"
ts=2026-05-11T11:38:34.494863675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35668"
ts=2026-05-11T11:38:36.048353079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47362"
level=debug ts=2026-05-11T11:38:41.894718069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:38:42.083934424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T11:39:05.812497694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:39:12.085218823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:39:29.009114027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34544"
level=debug ts=2026-05-11T11:39:30.477212936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:39:42.087281017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:39:45.08097774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33304"
level=debug ts=2026-05-11T11:40:07.078199588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:40:07.079834627Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:40:12.088947616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:40:42.09066111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:40:45.084777549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48900"
level=debug ts=2026-05-11T11:40:50.245976502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:41:12.092259028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T11:41:39.146489336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:41:42.093805962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:42:12.095660862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:42:15.202823549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43226"
ts=2026-05-11T11:42:29.017504262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38796"
level=debug ts=2026-05-11T11:42:39.117140275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:42:42.097777379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:42:59.019143747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56850"
ts=2026-05-11T11:43:12.099300422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:43:15.093214773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56850"
ts=2026-05-11T11:43:15.205183958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43694"
ts=2026-05-11T11:43:42.101165671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T11:43:48.026650014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:43:59.022192945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45314"
ts=2026-05-11T11:44:12.102381801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:44:15.096469362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35960"
ts=2026-05-11T11:44:15.208253702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50376"
ts=2026-05-11T11:44:42.103871385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:44:45.699003783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:44:45.700719545Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:44:59.024809047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58486"
ts=2026-05-11T11:45:12.105930961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:45:42.107608255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:45:42.83229218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:46:12.10952506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:46:15.102279939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55242"
ts=2026-05-11T11:46:15.214352766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48884"
level=debug ts=2026-05-11T11:46:24.967695346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:46:42.111941785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:46:45.103322491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57014"
level=debug ts=2026-05-11T11:47:03.969387244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:47:12.113850667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:47:42.115187084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:47:49.330744183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:48:12.116886083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:48:15.219312322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37984"
level=info ts=2026-05-11T11:48:17.737732697Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-11T11:48:17.742052977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:48:17.744491582Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:48:17.746720902Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:48:17.746784697Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:48:17.746796435Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-11T11:48:42.11873653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:48:49.139862152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:48:49.141825062Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:48:59.036009647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34068"
ts=2026-05-11T11:49:12.120022495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:49:15.109441557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39034"
ts=2026-05-11T11:49:29.037352825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54050"
level=debug ts=2026-05-11T11:49:35.66447893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:49:42.121730516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:50:12.122686794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:50:15.107626468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:50:15.112734259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37332"
ts=2026-05-11T11:50:42.124036123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:51:11.720980169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:51:12.126180073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:51:42.145312211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:51:52.321777774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:52:12.146627778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:52:26.629306294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:52:26.630995275Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:52:42.147646056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:52:45.236827827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52166"
ts=2026-05-11T11:53:12.1487379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T11:53:28.212204744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:53:42.150467083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:53:59.051777024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43112"
ts=2026-05-11T11:54:12.151961808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:54:15.128412244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47206"
ts=2026-05-11T11:54:15.240483732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39742"
level=debug ts=2026-05-11T11:54:27.683115276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:54:29.052939192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56478"
ts=2026-05-11T11:54:42.153485725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:55:04.757504659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:55:12.154992593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:55:15.244101734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33088"
ts=2026-05-11T11:55:42.156624203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:55:43.22494264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:55:45.133511315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50102"
ts=2026-05-11T11:55:59.056834762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52996"
ts=2026-05-11T11:56:12.157902244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:56:15.135057779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42492"
level=debug ts=2026-05-11T11:56:30.560246073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:56:30.562122638Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T11:56:42.159741575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:56:45.248936198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49836"
ts=2026-05-11T11:56:59.059735672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49362"
ts=2026-05-11T11:57:12.160960276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:57:30.753206532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:57:42.162021043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T11:57:45.252523201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48568"
ts=2026-05-11T11:58:12.163411601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T11:58:15.14109792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49858"
ts=2026-05-11T11:58:15.253879098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35036"
level=error ts=2026-05-11T11:58:17.7540749Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:58:17.756669257Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:58:17.758882911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T11:58:17.758919485Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T11:58:20.524355413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:58:29.064182749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51204"
ts=2026-05-11T11:58:42.164607471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:59:01.737744536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:59:12.165802662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:59:29.068139275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:33062"
level=debug ts=2026-05-11T11:59:36.808627689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:59:42.166894408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T11:59:59.06969249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51218"
ts=2026-05-11T12:00:12.16840769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:00:30.777199335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:00:30.779929706Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:00:42.170279117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:00:45.152754795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47420"
ts=2026-05-11T12:00:45.263196052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58128"
ts=2026-05-11T12:01:12.172246961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:01:15.265795631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37500"
level=debug ts=2026-05-11T12:01:28.009822091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:01:29.075185631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36648"
ts=2026-05-11T12:01:42.173559172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:02:12.175188714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:02:24.805260793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:02:29.079202997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35438"
ts=2026-05-11T12:02:42.176714737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:03:12.178042063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:03:15.161482573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43140"
ts=2026-05-11T12:03:15.271082861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54954"
level=debug ts=2026-05-11T12:03:19.219915314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:03:42.179732187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:03:45.163178912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50580"
ts=2026-05-11T12:03:59.082768825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43354"
level=debug ts=2026-05-11T12:04:10.225092443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:04:12.181087155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:04:15.274926239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57718"
ts=2026-05-11T12:04:42.182964854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:05:01.533605149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:05:01.535605765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:05:12.183923645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:05:15.16799866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37740"
level=debug ts=2026-05-11T12:05:39.036199788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:05:42.185670017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:05:45.279248696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59186"
ts=2026-05-11T12:06:12.186950417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:06:36.000446777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:06:42.188451626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:06:59.092003988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40836"
ts=2026-05-11T12:07:12.189732986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:07:24.975811361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:07:42.190953833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:07:45.174951379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48738"
ts=2026-05-11T12:07:45.286126655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57382"
ts=2026-05-11T12:07:59.094701481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49434"
level=debug ts=2026-05-11T12:08:10.447306147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:08:12.192639529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=error ts=2026-05-11T12:08:17.750004526Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:08:17.75185879Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:08:17.754162229Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:08:17.754202378Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:08:42.193967003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:08:45.178018729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42628"
ts=2026-05-11T12:08:45.290021554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34080"
level=debug ts=2026-05-11T12:08:53.168225472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:08:53.1699426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:09:12.19566414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:09:27.65636978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:09:42.19740881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:09:59.102627481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39330"
level=debug ts=2026-05-11T12:10:02.809212977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:10:12.199022224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:10:15.182061125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43172"
ts=2026-05-11T12:10:15.295137044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51700"
ts=2026-05-11T12:10:42.200529088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:10:50.661362285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:11:12.202608195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:11:29.108300584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39208"
ts=2026-05-11T12:11:42.203598867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:11:46.8084614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:12:12.204686018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:12:15.187494155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36922"
level=debug ts=2026-05-11T12:12:29.778694416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:12:29.780335961Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:12:42.206327044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:12:45.189164541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52208"
ts=2026-05-11T12:13:12.207645737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:13:15.190317167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60630"
ts=2026-05-11T12:13:15.303826177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47642"
level=debug ts=2026-05-11T12:13:22.107301908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:13:42.209475894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:13:45.191491657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33238"
level=debug ts=2026-05-11T12:14:04.027173733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:14:12.211340243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:14:15.19278421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50074"
ts=2026-05-11T12:14:29.118695441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44006"
ts=2026-05-11T12:14:42.212713648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:14:59.043860781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:15:12.214060796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:15:29.12199971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57318"
ts=2026-05-11T12:15:42.215366478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:15:44.715871345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:15:45.311937125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37436"
ts=2026-05-11T12:15:59.124013708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45588"
ts=2026-05-11T12:16:12.217154344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:16:15.313501779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35232"
level=debug ts=2026-05-11T12:16:29.663043539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:16:29.664993656Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:16:42.218579776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:16:45.315419301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:38882"
ts=2026-05-11T12:17:12.219837874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:17:15.316977774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:40166"
level=debug ts=2026-05-11T12:17:16.66408608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:17:42.221050817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:17:45.204862441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35090"
level=debug ts=2026-05-11T12:18:01.254174722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:18:12.222286449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=error ts=2026-05-11T12:18:17.751731909Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:18:17.753847692Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:18:17.755753775Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:18:17.755790415Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:18:42.224096522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:18:50.375019359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:19:12.225717305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:19:15.209733436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52712"
level=debug ts=2026-05-11T12:19:26.590208741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:19:42.226881576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:19:45.211406437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37562"
level=debug ts=2026-05-11T12:20:05.520410341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:20:05.522555987Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:20:12.228615459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:20:15.326458741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34468"
ts=2026-05-11T12:20:29.136948704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58630"
ts=2026-05-11T12:20:42.230371442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:20:51.042553191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:21:12.232131394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:21:15.329813232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33518"
ts=2026-05-11T12:21:29.140848751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35026"
level=debug ts=2026-05-11T12:21:40.385227845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:21:42.233871825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:22:12.236018457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:22:25.296622839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:22:42.237064868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:22:45.22129452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37404"
ts=2026-05-11T12:22:45.334439152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60438"
level=debug ts=2026-05-11T12:23:09.875556765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:23:12.2389597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:23:15.33587253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35630"
ts=2026-05-11T12:23:29.146872731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58214"
ts=2026-05-11T12:23:42.240672288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:23:53.665750928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:23:53.667326572Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:24:12.242447152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:24:15.224725807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41092"
ts=2026-05-11T12:24:15.338770706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44192"
ts=2026-05-11T12:24:29.150607911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39762"
level=debug ts=2026-05-11T12:24:29.212390412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:24:42.24395258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:24:45.340674086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56568"
level=debug ts=2026-05-11T12:25:07.527435108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:25:12.245130285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:25:42.247002197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:25:46.14127076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:25:59.155045466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45600"
ts=2026-05-11T12:26:12.248636957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:26:15.231301459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59580"
ts=2026-05-11T12:26:29.156532648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42516"
level=debug ts=2026-05-11T12:26:38.650220121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:26:42.249662179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:26:45.232422843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33054"
ts=2026-05-11T12:27:12.251401595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:27:22.493676927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:27:22.495280739Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:27:42.25311162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:27:45.349843214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53086"
ts=2026-05-11T12:27:59.160998402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49792"
ts=2026-05-11T12:28:12.255187001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:28:15.351196058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35656"
level=error ts=2026-05-11T12:28:17.749736528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:28:17.751398317Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:28:17.753116169Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:28:17.753145761Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:28:19.567322038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:28:42.257077448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:28:45.352492945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34648"
ts=2026-05-11T12:29:12.258675867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:29:12.513862653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:29:42.259805766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:29:45.241786974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60820"
level=debug ts=2026-05-11T12:29:53.413890807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:30:12.26084512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:30:15.358179011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49530"
level=debug ts=2026-05-11T12:30:32.051354174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:30:42.262366711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:30:59.17102121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42666"
level=debug ts=2026-05-11T12:31:10.859622049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:31:10.861585847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:31:12.263387649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:31:42.265432428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:31:57.84600852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:31:59.174079059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34528"
ts=2026-05-11T12:32:12.267772389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:32:42.268795601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:32:55.330762142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:32:59.177675294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35676"
ts=2026-05-11T12:33:12.269830486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:33:15.252971165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40970"
ts=2026-05-11T12:33:42.271087645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:33:46.183251756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:34:12.272534048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:34:29.182148369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41168"
level=debug ts=2026-05-11T12:34:30.263886486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:34:42.274326884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:35:12.275648962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:35:15.374516698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41260"
level=debug ts=2026-05-11T12:35:20.833269452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:35:20.835671016Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:35:29.185368227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50522"
ts=2026-05-11T12:35:42.276924208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:35:45.375693617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46268"
level=debug ts=2026-05-11T12:35:58.629238124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:36:12.278353495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:36:42.279838592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:36:55.089415882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:36:59.189988902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54152"
ts=2026-05-11T12:37:12.281203738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:37:15.37964221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:45202"
ts=2026-05-11T12:37:42.282730423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:37:45.265764532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41674"
level=debug ts=2026-05-11T12:37:46.645419221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:38:12.283740707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=error ts=2026-05-11T12:38:17.751021972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:38:17.753036844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:38:17.754839938Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:38:17.7548649Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:38:29.194952235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59560"
ts=2026-05-11T12:38:42.285094624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:38:44.475029555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:38:59.196275935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37910"
ts=2026-05-11T12:39:12.286701376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:39:23.110429963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:39:23.112076996Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:39:29.200399727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59560"
ts=2026-05-11T12:39:42.288462159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:39:45.387384839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58392"
ts=2026-05-11T12:39:59.202311547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54668"
level=debug ts=2026-05-11T12:40:00.531846401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:40:12.289937781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:40:15.273557867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37450"
ts=2026-05-11T12:40:42.291223274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:40:45.274673011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34862"
level=debug ts=2026-05-11T12:40:51.772195102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:40:59.206005132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50782"
ts=2026-05-11T12:41:12.293171015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:41:33.830131856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:41:42.295221864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:41:45.278393474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47794"
ts=2026-05-11T12:42:12.296293471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:42:29.210953053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39790"
level=debug ts=2026-05-11T12:42:32.211637754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:42:42.298072948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:42:59.212588847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38066"
ts=2026-05-11T12:43:12.300090015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:43:15.398208062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36350"
level=debug ts=2026-05-11T12:43:26.693113905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:43:26.694896827Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:43:42.301356839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:43:45.285475756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43044"
ts=2026-05-11T12:43:59.216869455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53054"
ts=2026-05-11T12:44:12.303187049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:44:17.539259842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:44:29.221102191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40490"
ts=2026-05-11T12:44:42.305236221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:44:45.28783972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42568"
ts=2026-05-11T12:44:45.403257855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48634"
level=debug ts=2026-05-11T12:45:01.354075549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:45:12.30692187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:45:42.307932022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:45:45.290739059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34882"
level=debug ts=2026-05-11T12:45:57.96155998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:46:12.309223762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:46:15.291655435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39186"
ts=2026-05-11T12:46:29.227782785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60290"
ts=2026-05-11T12:46:42.311398939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:46:43.514294455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:46:45.408499786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53922"
ts=2026-05-11T12:47:12.312774561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:47:15.409608542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47354"
level=debug ts=2026-05-11T12:47:21.564592951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:47:21.566537441Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:47:42.313882233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:48:12.314902516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:48:13.624283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:48:15.298194301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56000"
level=error ts=2026-05-11T12:48:17.749719612Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:48:17.751361163Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:48:17.752978676Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:48:17.753025645Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:48:29.234207928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54294"
ts=2026-05-11T12:48:42.315833635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:48:45.299371933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52338"
ts=2026-05-11T12:48:45.414293103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50418"
level=debug ts=2026-05-11T12:48:55.921296346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:49:12.316537963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:49:15.415223658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43082"
ts=2026-05-11T12:49:29.240805557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37264"
level=debug ts=2026-05-11T12:49:33.237787909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:49:42.318181877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:49:45.301763581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47054"
ts=2026-05-11T12:49:59.242782351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47022"
ts=2026-05-11T12:50:12.319346463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:50:15.302879549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44072"
level=debug ts=2026-05-11T12:50:20.399989401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:50:42.321340704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:50:45.419881803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35516"
level=debug ts=2026-05-11T12:50:57.924988049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:50:57.926719781Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:50:59.246414627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52868"
ts=2026-05-11T12:51:12.322900868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:51:15.420971654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44686"
ts=2026-05-11T12:51:42.324904253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:51:44.515973135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:51:45.422040188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49752"
ts=2026-05-11T12:51:59.249018847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35266"
ts=2026-05-11T12:52:12.325914725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:52:15.423308408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54728"
ts=2026-05-11T12:52:29.250411757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44038"
level=debug ts=2026-05-11T12:52:36.834936651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:52:42.327503374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:52:45.310702824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56468"
ts=2026-05-11T12:53:12.329332814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:53:15.312289061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60922"
ts=2026-05-11T12:53:15.426544607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39090"
ts=2026-05-11T12:53:29.253422681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58030"
level=debug ts=2026-05-11T12:53:37.83367367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:53:42.330999799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:53:45.313484692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59096"
ts=2026-05-11T12:54:12.332766177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T12:54:34.837584888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:54:42.333734689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:54:59.264709986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58324"
ts=2026-05-11T12:55:12.335143431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:55:26.547942675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:55:26.549659434Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:55:29.266775205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53002"
ts=2026-05-11T12:55:42.336224771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:55:45.436972526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:42350"
level=debug ts=2026-05-11T12:56:08.254525098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:56:12.337946295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:56:42.33923091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:56:45.321404534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43952"
ts=2026-05-11T12:56:45.439452193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58824"
level=debug ts=2026-05-11T12:56:50.925804185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:57:12.341142203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T12:57:42.342500583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:57:45.324039715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46980"
level=debug ts=2026-05-11T12:57:47.703855566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:58:12.343378362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T12:58:15.444481101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44144"
level=error ts=2026-05-11T12:58:17.750704852Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:58:17.752484748Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:58:17.754253762Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T12:58:17.754282107Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:58:29.275612177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36548"
level=debug ts=2026-05-11T12:58:31.951271372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:58:42.345199649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:58:59.278122177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41806"
ts=2026-05-11T12:59:12.346250846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:59:26.967617682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:59:26.969326653Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T12:59:42.352705612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T12:59:45.449141299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:45018"
ts=2026-05-11T13:00:12.354306428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:00:15.450561585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53708"
level=debug ts=2026-05-11T13:00:24.792600109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:00:29.284656269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44626"
ts=2026-05-11T13:00:42.355458063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:00:45.452381305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34950"
level=debug ts=2026-05-11T13:01:01.214190164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:01:12.356803877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:01:15.453742254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41574"
ts=2026-05-11T13:01:42.358409838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:01:55.460973448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:02:12.36035131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:02:15.457241012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59912"
ts=2026-05-11T13:02:42.361935316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:02:51.852998174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:02:59.292395465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39418"
ts=2026-05-11T13:03:12.363634734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:03:15.339643762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58482"
ts=2026-05-11T13:03:15.459362418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54436"
ts=2026-05-11T13:03:29.293745673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42406"
level=debug ts=2026-05-11T13:03:36.230711332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:03:36.232599561Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T13:03:42.365113471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:03:59.296059478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53672"
ts=2026-05-11T13:04:12.366835853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T13:04:15.46212252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:40386"
ts=2026-05-11T13:04:29.299448441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52816"
level=debug ts=2026-05-11T13:04:29.330525745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:04:42.368418398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:04:45.346186482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51528"
level=debug ts=2026-05-11T13:05:06.69134505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:05:12.369923996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:05:15.464501975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49856"
ts=2026-05-11T13:05:29.302493906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52630"
ts=2026-05-11T13:05:42.371880848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:05:45.349027892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50664"
ts=2026-05-11T13:05:45.465938156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49738"
level=debug ts=2026-05-11T13:05:52.217320304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:05:59.303929069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42400"
ts=2026-05-11T13:06:12.372883003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:06:15.467433347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34066"
level=debug ts=2026-05-11T13:06:42.077546697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:06:42.373945275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:07:12.375196749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T13:07:29.308443467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46850"
level=debug ts=2026-05-11T13:07:39.78050567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:07:39.782073261Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T13:07:42.37709689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:07:45.3562025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53980"
ts=2026-05-11T13:07:59.309655989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38298"
ts=2026-05-11T13:08:12.378285278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T13:08:15.3577182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36050"
level=error ts=2026-05-11T13:08:17.751235643Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T13:08:17.753051512Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T13:08:17.755025825Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T13:08:17.755056406Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T13:08:29.065749261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:08:42.379995577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T13:08:45.35948467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37924"
level=debug ts=2026-05-11T13:09:08.063373632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:09:12.381250504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:09:42.382477056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:09:45.480115995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:38496"
level=debug ts=2026-05-11T13:09:55.575085223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:10:12.383986613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:10:42.385195813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:10:47.74249445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:10:59.322717739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45266"
ts=2026-05-11T13:11:12.387031399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:11:36.960402908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:11:36.962134879Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T13:11:42.388967647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:12:12.390486162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:12:15.370773694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:32810"
ts=2026-05-11T13:12:15.487458528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55278"
level=debug ts=2026-05-11T13:12:23.167351044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:12:29.326683413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39428"
ts=2026-05-11T13:12:42.391538666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
ts=2026-05-11T13:12:45.488416072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43398"
ts=2026-05-11T13:12:59.328400805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58120"
ts=2026-05-11T13:13:12.392999461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:13:15.393970845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:13:15.490137534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51924"
ts=2026-05-11T13:13:42.394233762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
ts=2026-05-11T13:14:12.395283065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-28cb5960 10.128.0.88:7946"
level=debug ts=2026-05-11T13:14:12.399727197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:14:15.377683495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60250"
ts=2026-05-11T13:14:15.492506848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54790"
ts=2026-05-11T13:14:42.396806266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:15:12.398189604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:15:12.646709767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:15:15.49689879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44324"
ts=2026-05-11T13:15:42.399790958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:15:56.38122632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:15:56.382896728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T13:16:12.401389749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
ts=2026-05-11T13:16:15.383086592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34948"