level=info ts=2026-04-17T09:14:16.588872823Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T09:14:16.589124168Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T09:14:16.592405796Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T09:14:16.59269746Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T09:14:16.592784369Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.92 inf=eth0
level=debug ts=2026-04-17T09:14:16.593750456Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T09:14:16.593824814Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.92 inf=eth0
level=info ts=2026-04-17T09:14:16.594067476Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-2998cca2
ts=2026-04-17T09:14:16.594142122Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T09:14:16.594158697Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.92 advertisePort=7946
level=debug ts=2026-04-17T09:14:16.59417637Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.92 advertisePort=7946
level=info ts=2026-04-17T09:14:16.594337138Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T09:14:16.604214017Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T09:14:16.604234543Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:14:16.604247303Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.912193ms
level=error ts=2026-04-17T09:14:16.604257135Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T09:14:16.604268119Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T09:14:16.610812213Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:16.610831889Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T09:14:16.610967102Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-17T09:14:16.611043157Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-17T09:14:16.611054899Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-17T09:14:16.611075617Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-17T09:14:16.611103062Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-17T09:14:16.611179545Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-17T09:14:16.611425454Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-17T09:14:16.611440311Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-17T09:14:16.611446837Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-17T09:14:16.611453657Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T09:14:16.611531806Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T09:14:16.611583612Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T09:14:16.611608893Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T09:14:16.611751325Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T09:14:16.611765185Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T09:14:16.611802444Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T09:14:17.612390976Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-04-17T09:14:17.669734354Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:17.669758583Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T09:14:17.794186048Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T09:14:17.794220769Z caller=loki.go:599 msg="Loki started" startup_time=1.275269871s
ts=2026-04-17T09:14:20.569699658Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:20.569725335Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T09:14:22.794630899Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T09:14:22.794664172Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T09:14:24.988801958Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:24.988824049Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T09:14:31.657498131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:14:38.26191688Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:38.261944362Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T09:14:45.733264538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:14:55.713777217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43734"
ts=2026-04-17T09:14:56.862386499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52968"
ts=2026-04-17T09:14:57.381099777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:15:02.392263092Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.86:7946: dial tcp 10.129.0.86:7946: connect: connection refused"
ts=2026-04-17T09:15:02.39318344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.90:7946"
ts=2026-04-17T09:15:02.394062763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.85:7946"
ts=2026-04-17T09:15:02.395435163Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.91:7946: dial tcp 10.129.0.91:7946: connect: connection refused"
ts=2026-04-17T09:15:02.396149888Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.89:7946: dial tcp 10.129.0.89:7946: connect: connection refused"
ts=2026-04-17T09:15:02.396248886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.92:7946"
ts=2026-04-17T09:15:02.39626404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:55614"
ts=2026-04-17T09:15:02.396711622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.87:7946"
level=info ts=2026-04-17T09:15:02.397146642Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=45.792872115s
level=debug ts=2026-04-17T09:15:05.763865304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:15:07.002533802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:57890"
ts=2026-04-17T09:15:08.072923863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:57904"
ts=2026-04-17T09:15:27.382481454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:15:33.580446347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:15:34.554623999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55534"
ts=2026-04-17T09:15:43.306131102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39732"
ts=2026-04-17T09:15:57.383525459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:16:06.613583734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T09:16:06.615155336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:16:27.384819598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:16:37.005490248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:50308"
ts=2026-04-17T09:16:43.308451517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:35090"
level=debug ts=2026-04-17T09:16:49.698611723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:16:57.386023917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:17:27.386696278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:17:41.381593114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:17:57.387480416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:18:07.009686501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:38530"
ts=2026-04-17T09:18:13.313396591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:59258"
ts=2026-04-17T09:18:27.388377458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:18:34.562974278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45908"
ts=2026-04-17T09:18:43.314266534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39620"
level=debug ts=2026-04-17T09:18:54.576984849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:18:57.389261443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:19:13.315383166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:59760"
ts=2026-04-17T09:19:27.390551409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:19:34.56600009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54654"
ts=2026-04-17T09:19:37.013240605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:59648"
ts=2026-04-17T09:19:57.391805756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:20:04.567290705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39172"
level=debug ts=2026-04-17T09:20:05.211930549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:20:27.3934064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:20:34.568101569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45380"
ts=2026-04-17T09:20:57.394349537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:21:15.088933925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T09:21:15.090664118Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:21:27.395638504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:21:43.321057208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39446"
ts=2026-04-17T09:21:57.397073292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:22:04.90432356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:22:27.398377829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:22:34.572778764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57282"
ts=2026-04-17T09:22:43.323132861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:50604"
ts=2026-04-17T09:22:57.399348891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:23:03.198064571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:23:27.400219546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:23:34.575773517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58498"
level=debug ts=2026-04-17T09:23:39.278150793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:23:57.401631884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T09:24:22.794997477Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-17T09:24:22.796937596Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:24:22.798488547Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:24:22.80007543Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:24:22.800102212Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T09:24:22.80011434Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-17T09:24:23.695128628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:24:27.402747783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:24:43.327925105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:58756"
ts=2026-04-17T09:24:57.403443251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:25:07.025412744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:49682"
level=debug ts=2026-04-17T09:25:11.793368437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:25:11.795221111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:25:27.404492397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:25:37.026704789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:57042"
level=debug ts=2026-04-17T09:25:51.55113405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:25:57.405254937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:26:04.580971299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39936"
ts=2026-04-17T09:26:07.027978676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:53418"
ts=2026-04-17T09:26:13.331807819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36176"
ts=2026-04-17T09:26:27.4067393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:26:49.340588984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:26:57.407543453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:27:24.796417868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:27:27.408744044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:27:34.584481075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45402"
ts=2026-04-17T09:27:57.409637796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:28:12.567899366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:28:27.410355423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:28:34.586345997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41642"
ts=2026-04-17T09:28:43.338677162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:50618"
level=debug ts=2026-04-17T09:28:53.234291355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:28:53.23599114Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:28:57.411190137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:29:07.035899399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33122"
ts=2026-04-17T09:29:13.339469414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36290"
ts=2026-04-17T09:29:27.412539174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:29:34.589252147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54784"
ts=2026-04-17T09:29:37.037223137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36224"
level=debug ts=2026-04-17T09:29:37.113852595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:29:57.414168487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:30:04.590018757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48068"
ts=2026-04-17T09:30:07.038726132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:51552"
ts=2026-04-17T09:30:13.342989743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52000"
ts=2026-04-17T09:30:27.415335491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:30:32.676152957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:30:57.416258705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:31:04.593417202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60368"
ts=2026-04-17T09:31:13.34595732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:37134"
ts=2026-04-17T09:31:27.417132472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:31:34.661407844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:31:37.042872789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:41220"
ts=2026-04-17T09:31:57.418274281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:32:20.031028454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:32:27.419766167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:32:34.597801325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45672"
ts=2026-04-17T09:32:37.045207978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:55534"
level=debug ts=2026-04-17T09:32:56.322802253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:32:56.324437466Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:32:57.421329863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:33:07.046027776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:44310"
ts=2026-04-17T09:33:13.35170363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:47988"
ts=2026-04-17T09:33:27.422946196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:33:37.047064339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:56300"
level=debug ts=2026-04-17T09:33:50.001748003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:33:57.423896724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:34:04.601690191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33428"
level=error ts=2026-04-17T09:34:22.803896171Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:34:22.806080632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:34:22.807934604Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:34:22.807964409Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:34:27.425127592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:34:30.490382422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:34:37.049500766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:39462"
ts=2026-04-17T09:34:57.426725666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:35:04.603756918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55986"
level=debug ts=2026-04-17T09:35:20.707441451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:35:27.428125664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:35:57.429671079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:36:07.053239722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:55276"
level=debug ts=2026-04-17T09:36:11.847558611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:36:27.431287185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:36:37.054608978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36930"
level=debug ts=2026-04-17T09:36:55.022911383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:36:55.02440275Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:36:57.43266763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:37:04.607811544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37396"
ts=2026-04-17T09:37:13.361596927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36530"
ts=2026-04-17T09:37:27.433979243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:37:43.36324682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:46438"
level=debug ts=2026-04-17T09:37:54.769857865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:37:57.43483052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:38:04.610438745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53254"
ts=2026-04-17T09:38:13.364431554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:38102"
ts=2026-04-17T09:38:27.436078616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:38:48.399374325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:38:57.437329385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:39:04.612923175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35822"
ts=2026-04-17T09:39:07.060818105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36770"
ts=2026-04-17T09:39:27.438947658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:39:31.066026408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:39:37.06221964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:33180"
ts=2026-04-17T09:39:57.439792753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:40:04.616035845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48348"
level=debug ts=2026-04-17T09:40:08.921438476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:40:27.441384204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:40:34.617591649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47056"
ts=2026-04-17T09:40:37.064258102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:48124"
ts=2026-04-17T09:40:57.443294603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:41:02.850887432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:41:02.852668966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:41:04.619355397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55534"
ts=2026-04-17T09:41:07.065944224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:46528"
ts=2026-04-17T09:41:13.372505731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:41474"
ts=2026-04-17T09:41:27.444353547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:41:37.067154195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:34234"
level=debug ts=2026-04-17T09:41:50.649884774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:41:57.446259134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:42:13.375391577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44078"
ts=2026-04-17T09:42:27.447749547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:42:37.069853186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:38526"
level=debug ts=2026-04-17T09:42:46.89447249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:42:57.448938439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:43:04.626636397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52852"
ts=2026-04-17T09:43:07.071611346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:43178"
ts=2026-04-17T09:43:27.450853045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:43:34.628019981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54342"
level=debug ts=2026-04-17T09:43:45.518691432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:43:57.452080548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:44:04.629114343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56420"
level=error ts=2026-04-17T09:44:22.803145547Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:44:22.805114842Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:44:22.806941069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:44:22.80697496Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:44:24.948769706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:44:27.453429552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:44:34.630093784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53618"
ts=2026-04-17T09:44:57.454449524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:45:02.128413613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:45:02.130063108Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:45:27.455414247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:45:34.633539675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49096"
level=debug ts=2026-04-17T09:45:40.932817468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:45:57.456995727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:46:27.458821881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:46:34.63661631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60152"
level=debug ts=2026-04-17T09:46:38.318561508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:46:57.459893198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:47:04.637737903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57876"
ts=2026-04-17T09:47:07.083143881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:54256"
ts=2026-04-17T09:47:13.390462606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:60682"
level=debug ts=2026-04-17T09:47:23.742834192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:47:27.460867212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:47:57.462368346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:48:13.393184315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:59334"
level=debug ts=2026-04-17T09:48:14.912071723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:48:27.463989289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:48:34.642449346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35988"
ts=2026-04-17T09:48:37.087250282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:46386"
ts=2026-04-17T09:48:57.465583698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:49:04.643905728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58666"
level=debug ts=2026-04-17T09:49:05.360051009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:49:05.361875542Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:49:27.466715308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:49:37.090391955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:56506"
ts=2026-04-17T09:49:43.396508399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:54328"
level=debug ts=2026-04-17T09:49:54.32678754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:49:57.467951107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:50:27.469589697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:50:48.177509712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:50:57.471070394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:51:07.093987406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:51314"
ts=2026-04-17T09:51:27.471982242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:51:42.575237097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:51:57.473820873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:52:07.096816128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:46072"
ts=2026-04-17T09:52:27.4755244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:52:45.134421177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:52:57.476821683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:53:07.100077447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:48398"
ts=2026-04-17T09:53:27.478181752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:53:37.100886648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:53800"
level=debug ts=2026-04-17T09:53:40.629970183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:53:40.631996821Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:53:57.479928801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:54:07.102407394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:48598"
level=error ts=2026-04-17T09:54:22.814702695Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:54:22.819660023Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:54:22.82790265Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T09:54:22.827956167Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:54:27.481177397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:54:38.13216252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:54:57.482642272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:55:17.873061411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:55:27.483866547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T09:55:34.663306502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59828"
ts=2026-04-17T09:55:43.425144443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:55486"
ts=2026-04-17T09:55:57.485125054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:56:01.834234625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:56:27.486508556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:56:37.109422221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:60774"
ts=2026-04-17T09:56:43.427981469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:43954"
level=debug ts=2026-04-17T09:56:53.025996759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:56:57.48763297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:57:04.668292825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60900"
ts=2026-04-17T09:57:27.488733295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:57:37.112345439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:41020"
level=debug ts=2026-04-17T09:57:39.153483759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:57:39.155224051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T09:57:57.490043691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:58:04.671721734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35098"
ts=2026-04-17T09:58:27.491467271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T09:58:29.369870634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:58:34.672997761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44972"
ts=2026-04-17T09:58:57.493201934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T09:59:04.674433949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34284"
level=debug ts=2026-04-17T09:59:08.602249361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:59:27.494912041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T09:59:34.676028796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42788"
level=debug ts=2026-04-17T09:59:48.099034708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T09:59:57.496113243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:00:07.119684382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:44106"
level=debug ts=2026-04-17T10:00:23.232571319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:00:27.49772505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:00:34.679139887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60704"
ts=2026-04-17T10:00:37.121182566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:36704"
ts=2026-04-17T10:00:57.498499552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:00:58.355293302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:00:58.357021171Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T10:01:27.49959484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T10:01:34.681923615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44036"
level=debug ts=2026-04-17T10:01:35.020086545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:01:43.44297434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:34150"
ts=2026-04-17T10:01:57.500574529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T10:02:13.411778433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:02:13.445288585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:40632"
ts=2026-04-17T10:02:27.501352127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T10:02:57.50320168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:02:59.606900848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:03:04.685340863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33502"
ts=2026-04-17T10:03:27.504138474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:03:49.789154776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:03:57.507067909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:04:04.687945962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60154"
level=error ts=2026-04-17T10:04:22.803959959Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T10:04:22.806303482Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T10:04:22.808873081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T10:04:22.808900715Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T10:04:27.50838957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T10:04:43.452421561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:60374"
level=debug ts=2026-04-17T10:04:44.010817439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:04:44.012301913Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T10:04:57.509650453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T10:05:04.690476319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52018"
ts=2026-04-17T10:05:07.139464347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:40384"
ts=2026-04-17T10:05:27.510890521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
ts=2026-04-17T10:05:34.691536998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47782"
level=debug ts=2026-04-17T10:05:40.141265306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:05:43.454581549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45876"
ts=2026-04-17T10:05:57.512710605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T10:06:19.615060334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:06:27.513844873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T10:06:37.142868389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:49198"
ts=2026-04-17T10:06:57.514642796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:07:07.551899056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:07:13.458336954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36006"
ts=2026-04-17T10:07:27.516370339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:07:43.459458628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:55740"
ts=2026-04-17T10:07:57.517430021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2a8d9c9 10.129.0.90:7946"
level=debug ts=2026-04-17T10:07:59.827439313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:08:07.146386793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:47364"
ts=2026-04-17T10:08:27.522249383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:08:43.461690168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36984"
ts=2026-04-17T10:08:57.52413557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:08:58.945074656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:08:58.946751323Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T10:09:04.70067447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48636"
ts=2026-04-17T10:09:13.462803772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:51388"
ts=2026-04-17T10:09:27.525402928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:09:41.732197094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:09:57.528573645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T10:10:27.530508224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:10:42.768760379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:10:43.467563037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:38864"
ts=2026-04-17T10:10:57.531594193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
ts=2026-04-17T10:11:07.15512892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:51318"
level=debug ts=2026-04-17T10:11:17.979095074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:11:27.532511815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
ts=2026-04-17T10:11:34.707318897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36036"
ts=2026-04-17T10:11:57.533460663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:12:04.109574041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:12:04.708399836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53894"
ts=2026-04-17T10:12:13.471763204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:34626"