level=info ts=2026-04-19T09:44:17.171457689Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T09:44:17.171494838Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T09:44:17.173402529Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T09:44:17.173807504Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T09:44:17.173888926Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.77 inf=eth0
level=debug ts=2026-04-19T09:44:17.174469239Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T09:44:17.174551544Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.77 inf=eth0
level=info ts=2026-04-19T09:44:17.174558595Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-48043dc0
ts=2026-04-19T09:44:17.174655588Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T09:44:17.174672508Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.77 advertisePort=7946
level=debug ts=2026-04-19T09:44:17.174691777Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.77 advertisePort=7946
level=info ts=2026-04-19T09:44:17.174836377Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T09:44:17.179978685Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T09:44:17.179990858Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T09:44:17.179999381Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.165358ms
level=error ts=2026-04-19T09:44:17.180005712Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T09:44:17.180015761Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T09:44:17.18277884Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T09:44:17.182794577Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T09:44:17.191257004Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-19T09:44:17.191333862Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-19T09:44:17.191350533Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-19T09:44:17.19136455Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T09:44:17.191375881Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-19T09:44:17.191391803Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-19T09:44:17.191454326Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-19T09:44:17.191468778Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T09:44:17.191597306Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-19T09:44:17.191606829Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T09:44:17.19164864Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T09:44:17.191651166Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T09:44:17.191667144Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T09:44:17.19177346Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T09:44:17.191779649Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T09:44:17.191789328Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T09:44:18.192120923Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T09:44:18.373485341Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T09:44:18.373542626Z caller=loki.go:599 msg="Loki started" startup_time=1.278078076s
ts=2026-04-19T09:44:19.109558957Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T09:44:19.109584674Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:44:21.358253708Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T09:44:21.35828486Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T09:44:23.373975808Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T09:44:23.374026168Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T09:44:26.604605747Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T09:44:26.604633206Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:44:35.140968498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.77:7946"
ts=2026-04-19T09:44:35.141103716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49952"
ts=2026-04-19T09:44:35.143861918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.86:7946"
ts=2026-04-19T09:44:35.147078761Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
ts=2026-04-19T09:44:35.148717695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.87:7946"
ts=2026-04-19T09:44:35.15167506Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.85:7946: dial tcp 10.130.0.85:7946: connect: connection refused"
level=info ts=2026-04-19T09:44:35.151691505Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=17.971671893s
ts=2026-04-19T09:44:36.522654657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44672"
level=debug ts=2026-04-19T09:44:38.374568345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:44:40.075845498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:34874"
ts=2026-04-19T09:44:43.515263876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:46716"
level=debug ts=2026-04-19T09:44:58.30578173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:45:01.81470139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:45:16.272899634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:45514"
level=debug ts=2026-04-19T09:45:20.516271785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:45:25.963168973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53720"
ts=2026-04-19T09:45:31.816814062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:45:43.467799935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:45:46.180730108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54826"
ts=2026-04-19T09:45:55.966713536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:55232"
ts=2026-04-19T09:46:01.81938646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
level=debug ts=2026-04-19T09:46:17.430651647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T09:46:17.432614637Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T09:46:25.970200262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42778"
ts=2026-04-19T09:46:31.821084773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:46:46.281733446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:58088"
ts=2026-04-19T09:47:01.823395851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:47:02.966962851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:47:25.974580039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44892"
ts=2026-04-19T09:47:31.825612534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:47:46.191507795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:57464"
ts=2026-04-19T09:47:46.28627435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:57820"
ts=2026-04-19T09:47:55.977690914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57596"
ts=2026-04-19T09:48:01.827658831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T09:48:16.193196211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41660"
level=debug ts=2026-04-19T09:48:22.008786849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:48:31.829772018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:48:46.29064624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:48532"
ts=2026-04-19T09:48:55.984357211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38948"
ts=2026-04-19T09:49:01.832477722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T09:49:11.26856971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:49:31.835523521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T09:49:55.988885384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59728"
ts=2026-04-19T09:50:01.839119185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:50:06.999145139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T09:50:25.99149757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33962"
ts=2026-04-19T09:50:31.841612662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:50:46.208235978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:56154"
ts=2026-04-19T09:50:46.303416492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:52864"
ts=2026-04-19T09:51:01.842962499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
level=debug ts=2026-04-19T09:51:15.248573692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T09:51:15.25045359Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T09:51:16.209564591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50174"
ts=2026-04-19T09:51:16.306629466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:57194"
ts=2026-04-19T09:51:31.845664206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T09:51:46.210745458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60248"
ts=2026-04-19T09:51:46.310155893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:43682"
level=debug ts=2026-04-19T09:51:58.874485166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:52:01.848797374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T09:52:16.211884016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:47396"
ts=2026-04-19T09:52:16.313570875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:48252"
ts=2026-04-19T09:52:31.851481515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:52:58.162415046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:53:01.853356676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:53:31.855944524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T09:53:46.218958577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34512"
level=debug ts=2026-04-19T09:53:47.072236616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:53:56.005640352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38560"
ts=2026-04-19T09:54:01.860045903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T09:54:16.323328516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:54454"
level=info ts=2026-04-19T09:54:23.374685247Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T09:54:23.379672271Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T09:54:23.381764114Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T09:54:23.383655654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T09:54:23.383695244Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T09:54:23.383718744Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T09:54:31.862760466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T09:54:42.100792719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:54:46.326808144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:47656"
ts=2026-04-19T09:55:01.866982667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T09:55:26.013816776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44478"
ts=2026-04-19T09:55:31.87134636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:55:41.903452516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T09:55:41.905054404Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T09:55:56.017253571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53364"
ts=2026-04-19T09:56:01.874418622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T09:56:16.231662849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:37126"
ts=2026-04-19T09:56:16.334579083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:53516"
ts=2026-04-19T09:56:26.020444267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42062"
ts=2026-04-19T09:56:31.877267604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T09:56:36.478028465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:56:46.23271521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:57838"
ts=2026-04-19T09:57:01.88043287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T09:57:11.661090569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:57:31.882168179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:57:56.027788154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57236"
ts=2026-04-19T09:58:01.883215979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
level=debug ts=2026-04-19T09:58:13.637848416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:58:16.241584612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51076"
ts=2026-04-19T09:58:31.884915852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T09:58:56.03501364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59198"
ts=2026-04-19T09:59:01.885881017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
level=debug ts=2026-04-19T09:59:12.076283622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T09:59:16.248421575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:43748"
ts=2026-04-19T09:59:31.888687164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T09:59:48.883433281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T09:59:48.886335747Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:00:01.890571391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e848fed2 10.128.0.78:7946"
ts=2026-04-19T10:00:16.361957574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:54498"
level=debug ts=2026-04-19T10:00:23.365839916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:00:31.893385969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:00:46.365726102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:49416"
level=debug ts=2026-04-19T10:00:58.478249168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:01:01.896913315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:01:16.368537084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:48070"
ts=2026-04-19T10:01:31.900115295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:01:51.203341201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:01:56.060220917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:39718"
ts=2026-04-19T10:02:01.903621385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:02:16.375326401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:39932"
ts=2026-04-19T10:02:26.063470457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38668"
level=debug ts=2026-04-19T10:02:28.205227882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:02:31.294507529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:38972"
ts=2026-04-19T10:02:31.311516707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:38986"
ts=2026-04-19T10:02:31.906941321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:02:44.175907819Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e848fed2 (timeout reached)"
ts=2026-04-19T10:02:46.378722223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:34898"
ts=2026-04-19T10:02:47.176331507Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e848fed2 has failed, no acks received"
ts=2026-04-19T10:02:59.175674146Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e848fed2 (timeout reached)"
ts=2026-04-19T10:03:01.910326886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:03:02.176163077Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e848fed2 has failed, no acks received"
ts=2026-04-19T10:03:16.381750272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:38308"
level=debug ts=2026-04-19T10:03:20.722051362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:03:20.72387935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:03:31.912877031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70431857 10.128.0.78:7946"
ts=2026-04-19T10:03:56.083689614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33606"
ts=2026-04-19T10:04:01.914825771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70431857 10.128.0.78:7946"
ts=2026-04-19T10:04:16.388336613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:49528"
level=debug ts=2026-04-19T10:04:16.931915397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:04:23.387258957Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:04:23.389617851Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:04:23.391762781Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:04:23.391802982Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:04:31.916767327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70431857 10.128.0.78:7946"
ts=2026-04-19T10:04:47.267870982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51558"
ts=2026-04-19T10:05:01.919077651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:05:03.149808012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:05:17.268957524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39050"
ts=2026-04-19T10:05:26.089391333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:58616"
ts=2026-04-19T10:05:31.922164575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:05:41.231210189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:05:47.270144079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35728"
ts=2026-04-19T10:05:56.092927833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42662"
ts=2026-04-19T10:06:01.924769701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70431857 10.128.0.78:7946"
ts=2026-04-19T10:06:16.400519691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:48856"
ts=2026-04-19T10:06:26.09632768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:55552"
ts=2026-04-19T10:06:31.927265566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:06:40.570452203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:06:47.275253928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51120"
ts=2026-04-19T10:07:01.929451673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70431857 10.128.0.78:7946"
level=debug ts=2026-04-19T10:07:25.386432932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:07:25.388768105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:07:26.102113291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48930"
ts=2026-04-19T10:07:31.93170809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:07:46.406579292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:42700"
ts=2026-04-19T10:07:47.28072352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58188"
ts=2026-04-19T10:07:56.104923128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42874"
ts=2026-04-19T10:08:01.941422823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:08:24.012796855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:08:31.947590606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:08:56.11040205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51166"
ts=2026-04-19T10:09:01.950490164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:09:02.629930101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:09:09.175684439Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-70431857 (timeout reached)"
ts=2026-04-19T10:09:12.175745255Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-70431857 has failed, no acks received"
ts=2026-04-19T10:09:18.151052467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:55592"
ts=2026-04-19T10:09:18.167231202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:55606"
ts=2026-04-19T10:09:26.11321475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38368"
ts=2026-04-19T10:09:31.953777057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:09:32.176334242Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-70431857 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T10:09:34.175979927Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-70431857 (timeout reached)"
ts=2026-04-19T10:09:37.17592017Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-70431857 has failed, no acks received"
level=debug ts=2026-04-19T10:09:58.54881387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:10:01.95650319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
ts=2026-04-19T10:10:31.959263343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:10:37.248472027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:10:37.46614178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54804"
ts=2026-04-19T10:11:01.962450752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:11:24.133112125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:11:24.135070197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:11:31.964680719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
ts=2026-04-19T10:11:37.471278217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60642"
ts=2026-04-19T10:11:56.12565082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33790"
ts=2026-04-19T10:12:01.966088144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
ts=2026-04-19T10:12:07.472865396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42140"
level=debug ts=2026-04-19T10:12:08.942110116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:12:26.128730882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:40364"
ts=2026-04-19T10:12:31.967755435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
ts=2026-04-19T10:12:37.474314774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35956"
level=debug ts=2026-04-19T10:12:43.543134483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:12:46.429575969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:53918"
ts=2026-04-19T10:12:56.132136451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37830"
ts=2026-04-19T10:13:01.969908402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:13:26.134942495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:58250"
ts=2026-04-19T10:13:31.972122966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
level=debug ts=2026-04-19T10:13:43.138882734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:14:01.974919853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=error ts=2026-04-19T10:14:23.387997983Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:14:23.390412895Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:14:23.39244113Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:14:23.392473866Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:14:26.141760952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57794"
level=debug ts=2026-04-19T10:14:31.210152657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:14:31.976661362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f90917c 10.128.0.78:7946"
ts=2026-04-19T10:14:37.484640653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39890"
ts=2026-04-19T10:14:54.175012466Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1f90917c (timeout reached)"
ts=2026-04-19T10:14:57.17514715Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1f90917c has failed, no acks received"
ts=2026-04-19T10:15:01.978753852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:15:06.126498001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:40992"
ts=2026-04-19T10:15:06.142412862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41004"
ts=2026-04-19T10:15:16.439785616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:33656"
level=debug ts=2026-04-19T10:15:31.907889725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:15:31.909602884Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:15:31.981172124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
ts=2026-04-19T10:15:46.442818327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:59502"
ts=2026-04-19T10:15:56.089632208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52056"
ts=2026-04-19T10:16:01.982398687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
ts=2026-04-19T10:16:16.447434188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:44138"
ts=2026-04-19T10:16:26.152445261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:32988"
ts=2026-04-19T10:16:31.983663844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
level=debug ts=2026-04-19T10:16:32.210718307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:17:01.984543925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
ts=2026-04-19T10:17:16.456300731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:37330"
level=debug ts=2026-04-19T10:17:29.690914935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:17:31.986707786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:17:56.100225578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51100"
ts=2026-04-19T10:17:56.157779826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:47552"
ts=2026-04-19T10:18:01.988807101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
level=debug ts=2026-04-19T10:18:17.004588255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:18:31.990791948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:18:54.122142167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:18:56.10469236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58404"
ts=2026-04-19T10:18:56.161934082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:47902"
ts=2026-04-19T10:19:01.99344231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:19:26.164782555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:39272"
ts=2026-04-19T10:19:31.995432287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
level=debug ts=2026-04-19T10:19:45.439097314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:19:45.441416254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:19:46.468003054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:48954"
ts=2026-04-19T10:19:56.167664878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:46766"
ts=2026-04-19T10:20:01.997005552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
ts=2026-04-19T10:20:31.998744404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
level=debug ts=2026-04-19T10:20:44.154437619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:20:56.115187393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35908"
ts=2026-04-19T10:21:02.000433371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:21:26.116212541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34222"
ts=2026-04-19T10:21:26.172632529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42538"
ts=2026-04-19T10:21:32.002262458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-35f2ce79 10.128.0.78:7946"
level=debug ts=2026-04-19T10:21:39.582586033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:21:46.478397209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:44342"
ts=2026-04-19T10:22:02.004666143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:22:23.415649763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:22:26.178446221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56662"
ts=2026-04-19T10:22:32.007370573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:22:56.181879291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:54300"
ts=2026-04-19T10:23:02.009933822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:23:14.175701261Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-35f2ce79 (timeout reached)"
level=debug ts=2026-04-19T10:23:14.356158049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:23:17.175505498Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-35f2ce79 has failed, no acks received"
ts=2026-04-19T10:23:19.176321621Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-35f2ce79 (timeout reached)"
ts=2026-04-19T10:23:22.17667895Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-35f2ce79 has failed, no acks received"
ts=2026-04-19T10:23:32.012964895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:23:37.176189164Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-35f2ce79 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T10:23:39.175673779Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-35f2ce79 (timeout reached)"
ts=2026-04-19T10:23:42.175579389Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-35f2ce79 has failed, no acks received"
ts=2026-04-19T10:23:52.148361668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:45388"
ts=2026-04-19T10:23:52.162625407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:45398"
ts=2026-04-19T10:24:02.015544242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
level=debug ts=2026-04-19T10:24:14.517316768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:24:14.518902402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:24:16.4894944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:45828"
level=error ts=2026-04-19T10:24:23.386367197Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:24:23.388151139Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:24:23.389869394Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:24:23.389894448Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:24:32.018122419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:24:41.496551951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59686"
ts=2026-04-19T10:25:02.020614125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
level=debug ts=2026-04-19T10:25:11.905522906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:25:32.022634014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:26:00.056321819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:26:02.026032088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:26:16.498339689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:47554"
ts=2026-04-19T10:26:26.196490577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42806"
ts=2026-04-19T10:26:32.027848436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
level=debug ts=2026-04-19T10:27:00.884625255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:27:02.029083653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
ts=2026-04-19T10:27:26.20018996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56128"
ts=2026-04-19T10:27:32.031308854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:27:37.927997799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:28:02.033851838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
ts=2026-04-19T10:28:11.516420893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39720"
level=debug ts=2026-04-19T10:28:15.525795848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:28:15.527609114Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:28:16.507708731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:42834"
ts=2026-04-19T10:28:26.204430455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33724"
ts=2026-04-19T10:28:32.036375498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:28:41.518262922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50894"
level=debug ts=2026-04-19T10:28:50.919415498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:29:02.039825833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:29:26.20845902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:49248"
ts=2026-04-19T10:29:32.042816945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:29:49.161154828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:30:02.045231155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
ts=2026-04-19T10:30:16.516851016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:33076"
ts=2026-04-19T10:30:26.213209684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42610"
ts=2026-04-19T10:30:32.046217084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-acbf4c97 10.128.0.78:7946"
level=debug ts=2026-04-19T10:30:45.067532079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:31:02.049038935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:31:11.533201889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59980"
ts=2026-04-19T10:31:32.052023439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:31:45.95576419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:31:56.220998768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:34168"
ts=2026-04-19T10:32:02.055121593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:32:14.176333382Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-acbf4c97 (timeout reached)"
ts=2026-04-19T10:32:16.526017774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:34694"
ts=2026-04-19T10:32:17.176390846Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-acbf4c97 has failed, no acks received"
ts=2026-04-19T10:32:32.058719665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:32:44.981988187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:32:44.983734139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:32:46.528658089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:55984"
ts=2026-04-19T10:32:56.225633833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51616"
ts=2026-04-19T10:33:02.061442964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:33:16.531854499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:49024"
level=debug ts=2026-04-19T10:33:20.649257994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:33:24.148377566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58766"
ts=2026-04-19T10:33:24.163984249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58768"
ts=2026-04-19T10:33:32.064738195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:33:56.229246821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:56716"
ts=2026-04-19T10:34:02.066542964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
ts=2026-04-19T10:34:05.811351558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39198"
level=debug ts=2026-04-19T10:34:15.404542771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:34:23.386624131Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:34:23.388520658Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:34:23.390318521Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T10:34:23.390350913Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:34:32.069228041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:34:53.958357299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:34:56.23494413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:50476"
ts=2026-04-19T10:35:02.070969997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
ts=2026-04-19T10:35:32.073524877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:35:37.048993419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:36:02.076010352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
level=debug ts=2026-04-19T10:36:18.355714218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:36:18.35732608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:36:26.242611958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38308"
ts=2026-04-19T10:36:32.077993675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:36:54.677627459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:37:02.080911697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:37:05.828849366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42958"
ts=2026-04-19T10:37:16.548493723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:43794"
ts=2026-04-19T10:37:26.248208907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37198"
ts=2026-04-19T10:37:32.083008656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
ts=2026-04-19T10:37:35.830362236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:36530"
ts=2026-04-19T10:37:46.551460002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:41620"
level=debug ts=2026-04-19T10:37:48.490633716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:37:56.251684818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59366"
ts=2026-04-19T10:38:02.084752006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
level=debug ts=2026-04-19T10:38:28.718770483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:38:32.086894874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:39:02.088613812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
level=debug ts=2026-04-19T10:39:09.90398511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:39:32.091343807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:40:00.18231005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T10:40:00.184346693Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T10:40:02.094943929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:40:05.843130278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59338"
ts=2026-04-19T10:40:32.09759797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
ts=2026-04-19T10:40:35.844593433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39854"
level=debug ts=2026-04-19T10:40:45.714443825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:40:46.564714889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:59688"
ts=2026-04-19T10:41:02.09982357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
ts=2026-04-19T10:41:05.845372401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41672"
ts=2026-04-19T10:41:32.102651711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-kr47j-af63be0f 10.130.0.86:7946"
level=debug ts=2026-04-19T10:41:34.084418616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:41:35.84679185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:46872"
ts=2026-04-19T10:41:46.568749339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:34728"
ts=2026-04-19T10:42:02.104994427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33353645 10.128.0.78:7946"
ts=2026-04-19T10:42:05.84863713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52330"
ts=2026-04-19T10:42:16.571855692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:56944"
level=debug ts=2026-04-19T10:42:17.496961329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:42:32.107255323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
ts=2026-04-19T10:43:02.110699415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"
level=debug ts=2026-04-19T10:43:12.243549213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T10:43:32.11352107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djl6g-09ed0214 10.130.0.87:7946"