level=info ts=2026-04-19T22:28:45.870937007Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T22:28:45.870971247Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T22:28:45.872156324Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T22:28:45.872610357Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T22:28:45.872717008Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=debug ts=2026-04-19T22:28:45.874045699Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T22:28:45.874130161Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=info ts=2026-04-19T22:28:45.874165807Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-e3bf2622
ts=2026-04-19T22:28:45.874241989Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T22:28:45.874254277Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=debug ts=2026-04-19T22:28:45.874276245Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=info ts=2026-04-19T22:28:45.874398721Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T22:28:45.879079306Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T22:28:45.879094306Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T22:28:45.879102709Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.706325ms
level=error ts=2026-04-19T22:28:45.879110239Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T22:28:45.87912095Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T22:28:45.881722128Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T22:28:45.881739624Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T22:28:45.887767249Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-19T22:28:45.887747188Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-19T22:28:45.887762799Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T22:28:45.88777461Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-19T22:28:45.887803162Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-19T22:28:45.887851585Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-19T22:28:45.887859636Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-19T22:28:45.887867765Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T22:28:45.888135251Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-19T22:28:45.888149072Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T22:28:45.888214403Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T22:28:45.888202477Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T22:28:45.888233556Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T22:28:45.888398038Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T22:28:45.888411245Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T22:28:45.888428025Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T22:28:46.88924651Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T22:28:47.078845035Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T22:28:47.078899042Z caller=loki.go:599 msg="Loki started" startup_time=1.270914546s
ts=2026-04-19T22:28:47.552715557Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T22:28:47.552740614Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:28:49.81690137Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T22:28:49.81692664Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T22:28:52.07913679Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T22:28:52.07918912Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T22:28:55.60842357Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T22:28:55.608449914Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T22:29:02.184366759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:29:07.065278349Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-04-19T22:29:07.066778537Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
level=warn ts=2026-04-19T22:29:07.066797076Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="2 errors occurred:\n\t* Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused\n\t* Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused\n\n"
ts=2026-04-19T22:29:10.542266319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:49768"
ts=2026-04-19T22:29:15.354936011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33276"
level=debug ts=2026-04-19T22:29:21.009777452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:29:30.499301616Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-04-19T22:29:30.500732772Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
ts=2026-04-19T22:29:30.501782466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
ts=2026-04-19T22:29:30.502636327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
ts=2026-04-19T22:29:30.502686554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34490"
ts=2026-04-19T22:29:30.50369028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
ts=2026-04-19T22:29:30.504962018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.70:7946"
ts=2026-04-19T22:29:30.507089626Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
level=info ts=2026-04-19T22:29:30.507100131Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=44.62797444s
ts=2026-04-19T22:29:31.353012686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55232"
level=debug ts=2026-04-19T22:29:34.743330266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:29:43.186236812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:30:01.673139255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59068"
level=debug ts=2026-04-19T22:30:04.286975019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:30:07.867388716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41182"
ts=2026-04-19T22:30:13.188444804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:30:34.042369501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56694"
level=debug ts=2026-04-19T22:30:36.449805222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T22:30:36.451875206Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:30:37.869550067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:52036"
ts=2026-04-19T22:30:43.191083791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:31:01.678414954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55852"
ts=2026-04-19T22:31:13.19336168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:31:13.477679269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:31:31.680794287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59270"
ts=2026-04-19T22:31:37.872773745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:47870"
ts=2026-04-19T22:31:43.194803448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:32:04.048004692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50942"
ts=2026-04-19T22:32:07.875152762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:46260"
ts=2026-04-19T22:32:13.196806386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:32:26.376552331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:32:31.6838307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33774"
ts=2026-04-19T22:32:37.877799048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:42638"
ts=2026-04-19T22:32:43.198534487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
ts=2026-04-19T22:33:13.200184364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:33:21.467424657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:33:37.883840486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41496"
ts=2026-04-19T22:33:43.202219218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:34:01.689577685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44140"
ts=2026-04-19T22:34:04.056998114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35766"
ts=2026-04-19T22:34:13.204177077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:34:36.395297096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T22:34:37.887421329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:39614"
ts=2026-04-19T22:34:43.20596977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:35:04.061284006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60598"
ts=2026-04-19T22:35:07.888932007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59616"
ts=2026-04-19T22:35:13.207839209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:35:43.20978391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:35:48.553590662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T22:35:48.555433457Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:36:01.696441683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59920"
ts=2026-04-19T22:36:04.064990746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58030"
ts=2026-04-19T22:36:07.892962196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:57132"
ts=2026-04-19T22:36:13.211750741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:36:31.118970999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:36:31.698937754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54102"
ts=2026-04-19T22:36:37.895670912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34866"
ts=2026-04-19T22:36:43.213857367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:37:07.8978572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:53118"
ts=2026-04-19T22:37:13.21621732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:37:15.958664117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:37:31.702454405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57660"
ts=2026-04-19T22:37:43.21858035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:37:51.591030322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:38:13.220984079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:38:31.705975232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35586"
ts=2026-04-19T22:38:37.902252419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:37522"
ts=2026-04-19T22:38:43.223704796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:38:44.357718752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-19T22:38:52.079415733Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T22:38:52.081582192Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:38:52.083298298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:38:52.084846125Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:38:52.084880081Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T22:38:52.084893925Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T22:39:04.07757757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54054"
ts=2026-04-19T22:39:07.90491153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:59516"
ts=2026-04-19T22:39:13.225644901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:39:25.162293796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:39:25.163773337Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:39:34.078655026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53048"
ts=2026-04-19T22:39:43.227183105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:40:01.711701175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47432"
level=debug ts=2026-04-19T22:40:09.011730637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:40:13.228592902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:40:31.713700653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56044"
ts=2026-04-19T22:40:34.082081198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39532"
ts=2026-04-19T22:40:37.9098832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:48650"
ts=2026-04-19T22:40:43.230881047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:41:01.793098914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:41:04.083520728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40818"
ts=2026-04-19T22:41:13.232283718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
ts=2026-04-19T22:41:31.717986142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44044"
ts=2026-04-19T22:41:43.233480711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:41:58.826401169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:42:13.234745229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:42:37.918971104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:49088"
ts=2026-04-19T22:42:43.237109509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:42:45.811161363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:43:07.921737456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34550"
ts=2026-04-19T22:43:13.238477398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:43:28.224189524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:43:28.22635594Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:43:43.239665508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
ts=2026-04-19T22:44:07.925320128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:39604"
level=debug ts=2026-04-19T22:44:11.629968784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:44:13.241285544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:44:31.73191287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34928"
ts=2026-04-19T22:44:37.927304624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:37878"
ts=2026-04-19T22:44:43.243534013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:44:50.971343608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:45:13.24555696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:45:25.922767416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:45:34.104257415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57070"
ts=2026-04-19T22:45:43.247955393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:46:13.250313835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:46:17.480510095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:46:37.936210841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:49670"
ts=2026-04-19T22:46:43.252917912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:47:05.556546199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:47:05.559606396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:47:07.938920918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:35286"
ts=2026-04-19T22:47:13.255267474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:47:34.112923018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44902"
ts=2026-04-19T22:47:43.258063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:47:48.316652692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:48:07.942501459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:40988"
ts=2026-04-19T22:48:13.260143522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:48:43.262176494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:48:43.309936642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:48:52.089500972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:48:52.09317117Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:48:52.096575645Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:48:52.096637227Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:49:13.264786957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:49:18.095168278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:49:43.26705021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:49:55.731425627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:50:04.124030176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39774"
ts=2026-04-19T22:50:07.948200188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:37202"
ts=2026-04-19T22:50:13.269158712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:50:34.125493036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39516"
level=debug ts=2026-04-19T22:50:34.787158847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:50:34.789173686Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:50:43.271024099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:51:01.75289037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43180"
ts=2026-04-19T22:51:13.272785176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:51:25.752076444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:51:31.755571193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46596"
ts=2026-04-19T22:51:43.275966467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:52:13.277720445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:52:13.293296834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:52:43.279590467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:53:01.762032359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40610"
ts=2026-04-19T22:53:07.96116955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:45662"
ts=2026-04-19T22:53:13.281297633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-19aef6d0 10.129.0.76:7946"
level=debug ts=2026-04-19T22:53:14.011879894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:53:31.764614328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51192"
ts=2026-04-19T22:53:34.139489235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55520"
ts=2026-04-19T22:53:37.964009266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:38614"
ts=2026-04-19T22:53:43.286024661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:53:53.362075215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36032"
ts=2026-04-19T22:53:53.375443255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36046"
ts=2026-04-19T22:54:02.875030558Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-19aef6d0 (timeout reached)"
ts=2026-04-19T22:54:05.87519442Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-19aef6d0 has failed, no acks received"
ts=2026-04-19T22:54:13.288544221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:54:15.407143158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:54:31.779092021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58530"
ts=2026-04-19T22:54:43.290587901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e03ea06a 10.129.0.76:7946"
ts=2026-04-19T22:54:48.183328601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55250"
level=debug ts=2026-04-19T22:55:00.11496865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:55:00.116852443Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:55:01.78156986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50076"
ts=2026-04-19T22:55:13.292039835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e03ea06a 10.129.0.76:7946"
ts=2026-04-19T22:55:18.18506402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38038"
ts=2026-04-19T22:55:31.78402029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51194"
ts=2026-04-19T22:55:37.974129444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:40514"
level=debug ts=2026-04-19T22:55:40.801281714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:55:43.293865277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:55:48.186372076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40382"
ts=2026-04-19T22:56:13.305011323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:56:24.675219111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:56:43.309423138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e03ea06a 10.129.0.76:7946"
ts=2026-04-19T22:56:48.208353545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56268"
level=debug ts=2026-04-19T22:57:08.006829746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:57:13.311231983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T22:57:31.789992305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57854"
ts=2026-04-19T22:57:43.313557113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:57:48.212690847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50634"
ts=2026-04-19T22:58:01.792339645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55668"
level=debug ts=2026-04-19T22:58:06.114303058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T22:58:07.989656608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:49460"
ts=2026-04-19T22:58:13.315708678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:58:18.214196687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60644"
ts=2026-04-19T22:58:31.794969265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50030"
ts=2026-04-19T22:58:37.992452964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:53552"
ts=2026-04-19T22:58:43.317790865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T22:58:45.486447414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T22:58:45.488582317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:58:52.089841165Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:58:52.091956636Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:58:52.094054117Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T22:58:52.094081981Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T22:59:01.796626899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50588"
ts=2026-04-19T22:59:13.320162819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T22:59:18.218239254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35908"
ts=2026-04-19T22:59:43.323009683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T22:59:47.455747507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:00:13.325136776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:00:23.169115677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:00:43.327577843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:00:48.224024776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46310"
ts=2026-04-19T23:01:13.329731436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:01:20.635209876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:01:38.002498418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:54048"
ts=2026-04-19T23:01:43.332980238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:02:05.042181405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:02:07.875574055Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e03ea06a (timeout reached)"
ts=2026-04-19T23:02:08.004804789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34534"
ts=2026-04-19T23:02:10.875436968Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e03ea06a has failed, no acks received"
ts=2026-04-19T23:02:13.335659855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:02:17.874948252Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e03ea06a (timeout reached)"
ts=2026-04-19T23:02:20.875493805Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e03ea06a has failed, no acks received"
ts=2026-04-19T23:02:25.050762081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43708"
ts=2026-04-19T23:02:25.062994673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43720"
ts=2026-04-19T23:02:30.875607416Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e03ea06a as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T23:02:38.007501991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41942"
ts=2026-04-19T23:02:43.338009179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdc5bc91 10.129.0.76:7946"
level=debug ts=2026-04-19T23:02:51.099636836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:02:51.101620024Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:03:13.33942375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:03:36.041004838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:03:38.011250182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:52924"
ts=2026-04-19T23:03:43.340867956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdc5bc91 10.129.0.76:7946"
ts=2026-04-19T23:04:01.813227412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60136"
ts=2026-04-19T23:04:08.013750096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:40602"
ts=2026-04-19T23:04:13.342458609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:04:23.867963537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60616"
ts=2026-04-19T23:04:38.015938713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:51284"
level=debug ts=2026-04-19T23:04:38.152512896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:04:43.345054531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:05:13.346383911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdc5bc91 10.129.0.76:7946"
level=debug ts=2026-04-19T23:05:24.972440855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:05:43.347608181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdc5bc91 10.129.0.76:7946"
ts=2026-04-19T23:05:53.873190254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51926"
ts=2026-04-19T23:06:01.820780214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47798"
level=debug ts=2026-04-19T23:06:11.902887441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:06:13.349577046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:06:23.874185597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39374"
ts=2026-04-19T23:06:43.352292579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:06:51.23071081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:06:51.233502887Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:07:13.354039806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:07:29.953726508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:07:31.826425511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38822"
ts=2026-04-19T23:07:38.026575347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:38818"
ts=2026-04-19T23:07:43.356164877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:08:01.828421208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56908"
ts=2026-04-19T23:08:13.358607824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:08:24.465647705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:08:38.030242717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41910"
ts=2026-04-19T23:08:43.360285185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=error ts=2026-04-19T23:08:52.089493853Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:08:52.092101828Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:08:52.094671975Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:08:52.094706845Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:09:01.832946352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33664"
ts=2026-04-19T23:09:13.362605566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bdc5bc91 10.129.0.76:7946"
level=debug ts=2026-04-19T23:09:21.084523994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:09:38.034497727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:51384"
ts=2026-04-19T23:09:43.364361633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:10:05.578319394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:10:07.875493625Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-bdc5bc91 (timeout reached)"
ts=2026-04-19T23:10:10.876313646Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-bdc5bc91 has failed, no acks received"
ts=2026-04-19T23:10:12.048737014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38502"
ts=2026-04-19T23:10:12.059143815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38516"
ts=2026-04-19T23:10:13.366715537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:10:18.012008114Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-bdc5bc91 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T23:10:31.837200617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57648"
ts=2026-04-19T23:10:43.368675936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:11:01.510635817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:11:01.512605408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:11:13.370778757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:11:43.373047459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:11:54.217152967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:12:01.844576233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52692"
ts=2026-04-19T23:12:13.375383317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:12:38.046679086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:41600"
ts=2026-04-19T23:12:43.377289668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:12:48.436375499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:13:13.380040738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:13:31.46879629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:13:31.849353198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58790"
ts=2026-04-19T23:13:43.38234526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:13:56.627508246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59582"
ts=2026-04-19T23:14:08.05150155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:37880"
ts=2026-04-19T23:14:13.38458696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:14:25.327000332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:14:31.854243143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60876"
ts=2026-04-19T23:14:43.386434521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5bd3d2eb 10.129.0.76:7946"
ts=2026-04-19T23:14:56.63088781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52528"
level=debug ts=2026-04-19T23:15:11.90238042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:15:11.904300875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:15:13.38828165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5bd3d2eb 10.129.0.76:7946"
ts=2026-04-19T23:15:38.056302431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:34236"
ts=2026-04-19T23:15:43.390126945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:16:08.058371996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:42422"
level=debug ts=2026-04-19T23:16:13.116334543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:16:13.392155763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:16:38.060281167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:44712"
ts=2026-04-19T23:16:43.394861973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:16:56.640337524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43862"
ts=2026-04-19T23:17:01.86171928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48116"
level=debug ts=2026-04-19T23:17:10.645754313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:17:13.39687042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:17:43.399245933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:17:58.726697242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:18:08.068005475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:50700"
ts=2026-04-19T23:18:13.40137785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
ts=2026-04-19T23:18:43.403136193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
level=debug ts=2026-04-19T23:18:51.95046039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:18:52.087652514Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:18:52.089727773Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:18:52.092175856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T23:18:52.092218614Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:19:01.871607794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56038"
ts=2026-04-19T23:19:07.875684714Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5bd3d2eb (timeout reached)"
ts=2026-04-19T23:19:08.072854211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:36788"
ts=2026-04-19T23:19:10.875661521Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5bd3d2eb has failed, no acks received"
ts=2026-04-19T23:19:13.405442718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:19:23.022896198Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-5bd3d2eb as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-19T23:19:28.767979598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T23:19:28.769780209Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T23:19:31.873638912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45800"
ts=2026-04-19T23:19:36.062127685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40760"
ts=2026-04-19T23:19:36.072673025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40772"
ts=2026-04-19T23:19:43.407766332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-26895b89 10.129.0.76:7946"
level=debug ts=2026-04-19T23:20:09.522572691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:20:13.410177855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-wpb4q-883e22d7 10.130.0.68:7946"
ts=2026-04-19T23:20:38.077939749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:47962"
ts=2026-04-19T23:20:43.411679569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"
level=debug ts=2026-04-19T23:20:47.247130094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T23:21:08.079787834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.70:32816"
ts=2026-04-19T23:21:13.414005808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-9vtsm-16f8ee6c 10.130.0.70:7946"