level=info ts=2026-04-26T11:23:17.238163343Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-26T11:23:17.238186311Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-26T11:23:17.239111432Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-26T11:23:17.239371951Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-26T11:23:17.239438326Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.84 inf=eth0
level=debug ts=2026-04-26T11:23:17.240084956Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-26T11:23:17.240145869Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.84 inf=eth0
level=info ts=2026-04-26T11:23:17.240194038Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f0d3fae3
ts=2026-04-26T11:23:17.240260333Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-26T11:23:17.240270753Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.84 advertisePort=7946
level=debug ts=2026-04-26T11:23:17.240284211Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.84 advertisePort=7946
level=info ts=2026-04-26T11:23:17.240379155Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-26T11:23:17.244727169Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-26T11:23:17.244745578Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-26T11:23:17.244758785Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.380929ms
level=error ts=2026-04-26T11:23:17.244768589Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-26T11:23:17.244782847Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-26T11:23:17.247388348Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T11:23:17.247403982Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-26T11:23:17.253157838Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-26T11:23:17.253159771Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-26T11:23:17.253179142Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-26T11:23:17.253205886Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-26T11:23:17.253217283Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-26T11:23:17.253233828Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-26T11:23:17.253245135Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-26T11:23:17.253446208Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-26T11:23:17.253459158Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-26T11:23:17.253463868Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-26T11:23:17.25354137Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-26T11:23:17.25354147Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-26T11:23:17.253557585Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-26T11:23:17.253669797Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-26T11:23:17.25367616Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-26T11:23:17.253685583Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-26T11:23:18.254413722Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-26T11:23:18.410497272Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-26T11:23:18.410537971Z caller=loki.go:599 msg="Loki started" startup_time=1.239763964s
ts=2026-04-26T11:23:18.763826696Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T11:23:18.763850365Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:23:22.287369984Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T11:23:22.28738845Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-26T11:23:23.411198998Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-26T11:23:23.411244902Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-26T11:23:26.942532905Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-26T11:23:26.942555391Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-26T11:23:29.625868412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:23:34.98899759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43524"
ts=2026-04-26T11:23:37.256028465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43796"
ts=2026-04-26T11:23:40.51106162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37364"
ts=2026-04-26T11:23:41.068442361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.81:7946"
ts=2026-04-26T11:23:41.069099685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-26T11:23:41.06964206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.84:7946"
ts=2026-04-26T11:23:41.069666924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46422"
ts=2026-04-26T11:23:41.071094085Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.85:7946: dial tcp 10.130.0.85:7946: connect: connection refused"
level=info ts=2026-04-26T11:23:41.07110509Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.8263162s
level=debug ts=2026-04-26T11:23:41.9830715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:23:55.372932577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:57258"
level=debug ts=2026-04-26T11:23:59.975669367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:24:10.381787253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:24:26.287052148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:24:40.383408153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:24:54.807567782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-26T11:24:54.809187578Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:25:10.384160986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:25:23.047418037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36502"
ts=2026-04-26T11:25:40.385245638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:25:43.013008575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:26:10.386390121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:26:14.108685625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51422"
ts=2026-04-26T11:26:23.049752104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54428"
level=debug ts=2026-04-26T11:26:32.732650012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:26:40.387469149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:26:55.380713264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35112"
ts=2026-04-26T11:27:10.38890749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:27:14.11168725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60288"
ts=2026-04-26T11:27:25.38230018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:39924"
level=debug ts=2026-04-26T11:27:32.975766344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:27:40.390354045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:27:44.112816803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35692"
ts=2026-04-26T11:27:53.053564105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58784"
ts=2026-04-26T11:28:10.391693243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:28:14.113993225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50574"
ts=2026-04-26T11:28:40.393152105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:28:40.729092171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-26T11:28:55.385929645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:41274"
ts=2026-04-26T11:29:10.394262756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:29:14.11678151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36696"
ts=2026-04-26T11:29:25.387111015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58850"
ts=2026-04-26T11:29:40.395638595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:29:53.023380739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-26T11:29:53.025178805Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:30:10.396432688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:30:14.119032885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36996"
ts=2026-04-26T11:30:40.397213778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:30:41.509788554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:31:10.398882581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:31:27.619061334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:31:40.399864244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:31:53.064077013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55808"
ts=2026-04-26T11:32:10.401169112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:32:23.065728635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55042"
level=debug ts=2026-04-26T11:32:28.340829313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:32:40.402164167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:32:44.125900513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32826"
ts=2026-04-26T11:32:55.395000561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:32812"
level=debug ts=2026-04-26T11:33:10.214996063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:33:10.403679548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=info ts=2026-04-26T11:33:23.411814086Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-26T11:33:23.414719397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:33:23.416467226Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:33:23.418239885Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:33:23.418265013Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-26T11:33:23.418276553Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-26T11:33:40.404766924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:33:44.12872694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:54584"
level=debug ts=2026-04-26T11:33:45.851334476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:33:45.853127868Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:34:10.406551748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:34:25.399039554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:33404"
level=debug ts=2026-04-26T11:34:36.531395684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:34:40.407435507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:35:10.408499668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:35:19.523326267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:35:23.073309289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50136"
ts=2026-04-26T11:35:40.410017796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:35:53.074781537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33604"
ts=2026-04-26T11:36:10.411459964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:36:14.135008424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49850"
level=debug ts=2026-04-26T11:36:14.345322421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:36:25.405347183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58098"
ts=2026-04-26T11:36:40.412972057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:36:55.406382406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:44582"
ts=2026-04-26T11:37:10.41398944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:37:13.613938273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:37:40.415875299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:37:56.089323039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:37:56.091034636Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:38:10.41704481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:38:40.417981071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:38:51.066245173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:38:55.411621238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35136"
ts=2026-04-26T11:39:10.419267385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:39:40.421092475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:39:41.004283968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:40:10.422292495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:40:17.960845266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:40:23.088305029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54738"
ts=2026-04-26T11:40:40.423317186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:40:44.147519285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55148"
level=debug ts=2026-04-26T11:40:54.649521065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:41:10.424137892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:41:40.425589817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:41:44.15019391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38316"
level=debug ts=2026-04-26T11:41:48.249593586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:41:48.251722003Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:41:53.09294305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39454"
ts=2026-04-26T11:42:10.426853061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=debug ts=2026-04-26T11:42:39.04750681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:42:40.428009912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:42:44.153059878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47244"
ts=2026-04-26T11:42:53.096718836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33668"
ts=2026-04-26T11:43:10.429545291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
level=error ts=2026-04-26T11:43:23.422576249Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:43:23.424569885Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:43:23.42673362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:43:23.426760591Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-26T11:43:33.237702522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:43:40.430905349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:43:53.099463214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52434"
ts=2026-04-26T11:44:10.43246648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:44:34.983137197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:44:40.4339711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:45:10.43539873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:45:23.103614293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43422"
level=debug ts=2026-04-26T11:45:30.50206538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:45:40.43700787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:45:55.429991099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35856"
ts=2026-04-26T11:46:10.438518176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:46:18.18683643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:46:18.188832456Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:46:23.105883865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52134"
ts=2026-04-26T11:46:40.440146912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:46:53.107524033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34142"
level=debug ts=2026-04-26T11:47:01.274537448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:47:10.441755613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:47:23.108708923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42190"
level=debug ts=2026-04-26T11:47:37.030087682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:47:40.442727737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:48:10.44441692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:48:18.841732557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:48:23.112058852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56658"
ts=2026-04-26T11:48:25.435956679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:52174"
ts=2026-04-26T11:48:40.445686137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:48:44.17062713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60858"
ts=2026-04-26T11:48:53.113456295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42768"
ts=2026-04-26T11:48:55.360110706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53918"
ts=2026-04-26T11:48:55.369414649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:53926"
level=debug ts=2026-04-26T11:49:00.129735112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:49:09.241331184Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e86be1f0 (timeout reached)"
ts=2026-04-26T11:49:10.447020691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e86be1f0 10.130.0.83:7946"
ts=2026-04-26T11:49:12.241435821Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e86be1f0 has failed, no acks received"
ts=2026-04-26T11:49:25.438092827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53256"
ts=2026-04-26T11:49:29.240955887Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e86be1f0 (timeout reached)"
ts=2026-04-26T11:49:32.241814016Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e86be1f0 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T11:49:32.241858401Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e86be1f0 has failed, no acks received"
ts=2026-04-26T11:49:40.448044108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45f643e4 10.130.0.83:7946"
ts=2026-04-26T11:49:44.549513992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49786"
ts=2026-04-26T11:49:53.11726206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59844"
ts=2026-04-26T11:49:55.439850146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:59900"
level=debug ts=2026-04-26T11:49:57.88337186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:49:57.884964716Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:50:10.449632662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:50:14.550800383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55354"
ts=2026-04-26T11:50:23.118757057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57376"
ts=2026-04-26T11:50:40.450760691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T11:50:54.518856658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:50:55.442174309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38990"
ts=2026-04-26T11:51:10.451781654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45f643e4 10.130.0.83:7946"
ts=2026-04-26T11:51:14.553253127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:41312"
ts=2026-04-26T11:51:23.121425998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55324"
level=debug ts=2026-04-26T11:51:34.99127536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:51:40.453186373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:52:10.454705596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:52:20.78445058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:52:23.124068242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44724"
ts=2026-04-26T11:52:40.456026957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45f643e4 10.130.0.83:7946"
ts=2026-04-26T11:52:44.557552384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:39548"
ts=2026-04-26T11:52:55.448245736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:59744"
level=debug ts=2026-04-26T11:53:10.174539788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:53:10.457603292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45f643e4 10.130.0.83:7946"
ts=2026-04-26T11:53:14.559017416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36874"
ts=2026-04-26T11:53:23.126916678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53448"
level=error ts=2026-04-26T11:53:23.423332396Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:53:23.42540085Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:53:23.427338486Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T11:53:23.427369357Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:53:40.458885358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:53:53.12833187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36562"
level=debug ts=2026-04-26T11:53:59.628310087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:53:59.631020841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:54:10.462746949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:54:40.463660131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45f643e4 10.130.0.83:7946"
ts=2026-04-26T11:54:53.130844567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59244"
level=debug ts=2026-04-26T11:55:01.66745235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:55:10.465456791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:55:14.56470307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40410"
ts=2026-04-26T11:55:25.454732033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:41076"
ts=2026-04-26T11:55:40.466991285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:55:51.465362513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:56:10.46869173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:56:25.45700498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53914"
level=debug ts=2026-04-26T11:56:38.505043239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:56:40.46966329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:56:44.569847476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:60518"
ts=2026-04-26T11:57:10.471268456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T11:57:14.542639663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:57:40.472530386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:57:44.572982968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43712"
level=debug ts=2026-04-26T11:58:02.671622845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T11:58:02.673538429Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T11:58:10.473450851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:58:24.241826062Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-45f643e4 (timeout reached)"
ts=2026-04-26T11:58:27.241854588Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-45f643e4 has failed, no acks received"
ts=2026-04-26T11:58:27.548381312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45292"
ts=2026-04-26T11:58:27.557043457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:45302"
ts=2026-04-26T11:58:34.241847992Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-45f643e4 (timeout reached)"
ts=2026-04-26T11:58:37.241220758Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-45f643e4 has failed, no acks received"
ts=2026-04-26T11:58:39.242340273Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-45f643e4 (timeout reached)"
level=debug ts=2026-04-26T11:58:39.58926756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:58:40.475217791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T11:58:42.24201229Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-45f643e4 has failed, no acks received"
ts=2026-04-26T11:58:47.231774558Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-45f643e4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T11:59:10.47605669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T11:59:15.858741081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:57338"
ts=2026-04-26T11:59:23.142897794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51950"
ts=2026-04-26T11:59:25.465886918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35798"
level=debug ts=2026-04-26T11:59:28.901269061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T11:59:40.477662286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c48ee688 10.130.0.83:7946"
ts=2026-04-26T12:00:10.478763746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c48ee688 10.130.0.83:7946"
ts=2026-04-26T12:00:23.146298688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39616"
level=debug ts=2026-04-26T12:00:26.935576141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:00:40.480268023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:00:45.863116987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49602"
ts=2026-04-26T12:00:53.14797753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49732"
level=debug ts=2026-04-26T12:01:01.462101009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:01:10.481625274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c48ee688 10.130.0.83:7946"
ts=2026-04-26T12:01:25.471078393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36730"
ts=2026-04-26T12:01:40.483378925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:01:40.638875712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:01:40.640600081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:02:10.484429625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:02:15.867282424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37206"
level=debug ts=2026-04-26T12:02:21.392161177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:02:40.486052796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:02:45.869081236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36082"
ts=2026-04-26T12:02:53.153656085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49512"
ts=2026-04-26T12:02:55.474204346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:48536"
level=debug ts=2026-04-26T12:03:09.366147894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:03:10.487337048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:03:23.154818895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51948"
level=error ts=2026-04-26T12:03:23.422244496Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:03:23.424063338Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:03:23.425673788Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:03:23.425699623Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:03:25.475493772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35684"
ts=2026-04-26T12:03:40.48877517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:03:55.186948179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:04:10.490167809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c48ee688 10.130.0.83:7946"
ts=2026-04-26T12:04:40.491528442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c48ee688 10.130.0.83:7946"
ts=2026-04-26T12:04:45.874590849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:55466"
level=debug ts=2026-04-26T12:04:51.117919711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:05:10.492968549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:05:15.876151703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43068"
level=debug ts=2026-04-26T12:05:29.952406254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:05:29.954103212Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:05:40.496360015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:05:53.161803385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49972"
ts=2026-04-26T12:05:55.480907995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:37612"
ts=2026-04-26T12:06:10.497522856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:06:11.819843267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:06:40.499182223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:06:45.88007737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43918"
level=debug ts=2026-04-26T12:06:49.264318593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:07:04.241207142Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c48ee688 (timeout reached)"
ts=2026-04-26T12:07:07.240949615Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c48ee688 has failed, no acks received"
ts=2026-04-26T12:07:10.500237913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:07:19.474701186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42998"
ts=2026-04-26T12:07:19.482965173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43012"
ts=2026-04-26T12:07:24.241811121Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c48ee688 (timeout reached)"
ts=2026-04-26T12:07:27.233240936Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c48ee688 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T12:07:27.241794918Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c48ee688 has failed, no acks received"
ts=2026-04-26T12:07:40.501286008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8cd09208 10.130.0.83:7946"
level=debug ts=2026-04-26T12:07:48.874491825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:07:53.166747075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38360"
ts=2026-04-26T12:07:55.486144297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36262"
ts=2026-04-26T12:08:10.502880465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:08:27.11493765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:08:29.741994273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42388"
ts=2026-04-26T12:08:40.504279165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:08:53.168911732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44526"
level=debug ts=2026-04-26T12:09:07.875021388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:09:07.876709938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:09:10.505967558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:09:25.489711725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:60778"
ts=2026-04-26T12:09:40.507123168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8cd09208 10.130.0.83:7946"
level=debug ts=2026-04-26T12:10:08.941245029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:10:10.508711278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:10:25.493269801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:59102"
ts=2026-04-26T12:10:29.748138002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34762"
ts=2026-04-26T12:10:40.510123064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:10:46.697793883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:11:10.511172605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:11:40.512803896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:11:44.544733747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:12:10.513784142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8cd09208 10.130.0.83:7946"
ts=2026-04-26T12:12:40.515351479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8cd09208 10.130.0.83:7946"
level=debug ts=2026-04-26T12:12:41.132276571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:12:55.500692112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:55794"
ts=2026-04-26T12:13:10.516416898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=error ts=2026-04-26T12:13:23.422400363Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:13:23.424413271Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:13:23.426265043Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:13:23.426294028Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-26T12:13:27.246144604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:13:27.247974185Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:13:40.517776692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:13:53.181373861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53210"
ts=2026-04-26T12:14:10.518745162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:14:23.182736782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37014"
level=debug ts=2026-04-26T12:14:28.317652755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:14:40.519771755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8cd09208 10.130.0.83:7946"
ts=2026-04-26T12:14:53.184144676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43190"
level=debug ts=2026-04-26T12:15:07.949421577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:15:10.521256844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:15:23.185790551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56588"
ts=2026-04-26T12:15:40.522306552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:16:05.263245086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:16:10.523801749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:16:23.187281991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47928"
ts=2026-04-26T12:16:40.525513978Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-8cd09208 failed: dial tcp 10.130.0.83:7946: connect: connection refused"
ts=2026-04-26T12:16:49.240945379Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8cd09208 (timeout reached)"
ts=2026-04-26T12:16:52.241643557Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8cd09208 has failed, no acks received"
ts=2026-04-26T12:16:53.188442475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56142"
level=debug ts=2026-04-26T12:17:02.25229463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:17:10.526799457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:17:23.458620696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47146"
ts=2026-04-26T12:17:23.466829569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47154"
ts=2026-04-26T12:17:25.512668306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:37734"
ts=2026-04-26T12:17:40.527720989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:17:53.191194441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36812"
ts=2026-04-26T12:17:55.513946902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35078"
level=debug ts=2026-04-26T12:17:58.990716998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:17:58.992402473Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:18:10.529329833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:18:36.464553714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:18:40.530767414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:18:55.515835137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58862"
ts=2026-04-26T12:19:10.531591771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:19:33.461435472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:19:40.533076473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04f720fb 10.130.0.83:7946"
ts=2026-04-26T12:19:52.157570943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47934"
ts=2026-04-26T12:19:55.518143874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:52314"
ts=2026-04-26T12:20:10.534716523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:20:23.19919882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56148"
ts=2026-04-26T12:20:25.519629728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:41566"
level=debug ts=2026-04-26T12:20:32.029509916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:20:40.536136268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:20:52.16042848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42732"
level=debug ts=2026-04-26T12:21:07.008335668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:21:10.537314157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:21:22.161377661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:50252"
ts=2026-04-26T12:21:40.53866263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:21:52.313024364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:21:52.314463299Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:21:55.523984811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43082"
ts=2026-04-26T12:22:10.539806144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04f720fb 10.130.0.83:7946"
level=debug ts=2026-04-26T12:22:27.778289427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:22:40.540781047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04f720fb 10.130.0.83:7946"
ts=2026-04-26T12:22:52.165585722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40898"
ts=2026-04-26T12:22:55.526732779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:45982"
level=debug ts=2026-04-26T12:23:06.413248704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:23:10.542413393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:23:22.166791464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32810"
level=error ts=2026-04-26T12:23:23.424081346Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:23:23.425947046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:23:23.427746428Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:23:23.427771473Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:23:25.527690258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38948"
ts=2026-04-26T12:23:40.543465379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04f720fb 10.130.0.83:7946"
level=debug ts=2026-04-26T12:24:05.057634232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:24:10.544492681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:24:22.169187049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:33260"
ts=2026-04-26T12:24:23.20895867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49532"
ts=2026-04-26T12:24:40.545509345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04f720fb 10.130.0.83:7946"
level=debug ts=2026-04-26T12:25:02.166380441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:25:10.546410635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:25:16.48112793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36220"
ts=2026-04-26T12:25:16.489550599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:36226"
ts=2026-04-26T12:25:19.241516304Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-04f720fb (timeout reached)"
ts=2026-04-26T12:25:22.241114295Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-04f720fb has failed, no acks received"
ts=2026-04-26T12:25:29.241202543Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-04f720fb (timeout reached)"
ts=2026-04-26T12:25:32.241807818Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-04f720fb has failed, no acks received"
ts=2026-04-26T12:25:40.547381819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:25:42.23223644Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-04f720fb as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T12:25:53.212355447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40076"
ts=2026-04-26T12:25:55.533036578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:57244"
level=debug ts=2026-04-26T12:25:57.076180593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:25:57.077782286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:26:10.548747401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d00cecca 10.130.0.83:7946"
ts=2026-04-26T12:26:40.550316116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:26:48.630687117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:26:53.214919804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51646"
ts=2026-04-26T12:27:10.551193771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:27:25.537642516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:56416"
level=debug ts=2026-04-26T12:27:27.622212271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:27:40.552803047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:27:55.539203446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58734"
ts=2026-04-26T12:28:10.553889257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:28:11.975400235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:28:38.194633878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43320"
ts=2026-04-26T12:28:40.555113984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d00cecca 10.130.0.83:7946"
level=debug ts=2026-04-26T12:29:03.974276799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:29:10.556157595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:29:25.542403957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53176"
ts=2026-04-26T12:29:40.557843425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:29:53.221689161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52412"
level=debug ts=2026-04-26T12:29:53.855565651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:29:53.857394402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:30:10.559387932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d00cecca 10.130.0.83:7946"
ts=2026-04-26T12:30:25.545058582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35088"
ts=2026-04-26T12:30:38.20019892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38622"
ts=2026-04-26T12:30:40.561144331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d00cecca 10.130.0.83:7946"
level=debug ts=2026-04-26T12:30:52.42863005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:31:10.562643455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:31:38.203147635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37498"
ts=2026-04-26T12:31:40.563806052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d00cecca 10.130.0.83:7946"
level=debug ts=2026-04-26T12:31:42.900437749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:31:55.548331331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:56088"
ts=2026-04-26T12:32:08.204891886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:51062"
ts=2026-04-26T12:32:10.565243113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:32:25.549352745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35840"
level=debug ts=2026-04-26T12:32:32.898043498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:32:40.566742842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:32:44.241649414Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d00cecca (timeout reached)"
ts=2026-04-26T12:32:47.240939676Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d00cecca has failed, no acks received"
ts=2026-04-26T12:32:53.228738275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39814"
ts=2026-04-26T12:32:55.55015817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:48782"
ts=2026-04-26T12:32:58.47689652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35940"
ts=2026-04-26T12:32:58.484871586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35952"
ts=2026-04-26T12:32:59.241320474Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d00cecca (timeout reached)"
ts=2026-04-26T12:33:02.240888972Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d00cecca has failed, no acks received"
level=debug ts=2026-04-26T12:33:07.90787748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:33:10.568553582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cbe68d95 10.130.0.83:7946"
ts=2026-04-26T12:33:23.230331254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56732"
level=error ts=2026-04-26T12:33:23.424244035Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:33:23.426636921Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:33:23.428684055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:33:23.428729015Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:33:40.569559409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:33:55.545972503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:33:55.547833366Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:34:10.570958956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:34:31.70056309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:34:40.571885116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:35:10.572983246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:35:23.235418506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49708"
level=debug ts=2026-04-26T12:35:29.262895439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:35:40.574389106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:35:53.236740672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49204"
ts=2026-04-26T12:36:04.240728583Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cbe68d95 (timeout reached)"
ts=2026-04-26T12:36:07.241408639Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cbe68d95 has failed, no acks received"
ts=2026-04-26T12:36:09.242003519Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cbe68d95 (timeout reached)"
ts=2026-04-26T12:36:10.57585649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:36:12.24220817Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cbe68d95 has failed, no acks received"
level=debug ts=2026-04-26T12:36:18.776421599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:36:22.110096924Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cbe68d95 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T12:36:23.238539169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57106"
ts=2026-04-26T12:36:36.471851028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38090"
ts=2026-04-26T12:36:36.480444768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:38092"
ts=2026-04-26T12:36:40.576928714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:37:01.770287284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:37:10.57826254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:37:13.781438753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:56556"
ts=2026-04-26T12:37:25.562053648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:44446"
level=debug ts=2026-04-26T12:37:36.939968587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:37:36.941462254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:37:40.579192289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57ee6a24 10.130.0.83:7946"
ts=2026-04-26T12:38:10.58043972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:38:18.793061447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:38:23.245582721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34874"
ts=2026-04-26T12:38:25.563817806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:42330"
ts=2026-04-26T12:38:40.581883462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:38:43.785760139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:58952"
ts=2026-04-26T12:38:53.246359217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51440"
level=debug ts=2026-04-26T12:39:01.130557152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:39:10.583456906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57ee6a24 10.130.0.83:7946"
ts=2026-04-26T12:39:13.787323732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35886"
ts=2026-04-26T12:39:23.247710043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51782"
ts=2026-04-26T12:39:40.584518462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:39:43.788898522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42844"
level=debug ts=2026-04-26T12:39:56.060589539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:40:10.585407031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:40:32.519295146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:40:40.586457309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:40:55.569714335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:47768"
level=debug ts=2026-04-26T12:41:08.655630465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:41:08.657449538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:41:10.587537594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57ee6a24 10.130.0.83:7946"
ts=2026-04-26T12:41:13.791977822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34406"
ts=2026-04-26T12:41:23.252216234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55654"
ts=2026-04-26T12:41:25.571135565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:40674"
ts=2026-04-26T12:41:40.588908565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:41:43.793582629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:37408"
level=debug ts=2026-04-26T12:41:44.804120182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:41:53.25319332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39196"
ts=2026-04-26T12:42:10.589709031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:42:13.794934063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:43686"
ts=2026-04-26T12:42:23.25438063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52202"
ts=2026-04-26T12:42:40.591288032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:42:46.696670124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:42:55.57510602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43612"
ts=2026-04-26T12:43:10.59227923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-57ee6a24 10.130.0.83:7946"
level=error ts=2026-04-26T12:43:23.42165724Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:43:23.423378327Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:43:23.425023868Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:43:23.425051426Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:43:25.576794954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36534"
ts=2026-04-26T12:43:40.593464088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:43:44.242187422Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-57ee6a24 (timeout reached)"
level=debug ts=2026-04-26T12:43:46.302528107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:43:47.241940608Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-57ee6a24 has failed, no acks received"
ts=2026-04-26T12:43:53.259329638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51442"
ts=2026-04-26T12:43:54.241969555Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-57ee6a24 (timeout reached)"
ts=2026-04-26T12:43:55.578159126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38194"
ts=2026-04-26T12:43:57.241658311Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-57ee6a24 has failed, no acks received"
ts=2026-04-26T12:44:07.242862219Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-57ee6a24 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-26T12:44:09.241494594Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-57ee6a24 (timeout reached)"
ts=2026-04-26T12:44:10.594446938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:44:12.242016564Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-57ee6a24 has failed, no acks received"
ts=2026-04-26T12:44:23.260941874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51298"
ts=2026-04-26T12:44:25.579029483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:49856"
ts=2026-04-26T12:44:40.595817997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
level=debug ts=2026-04-26T12:44:46.584666534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:44:53.262516402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54100"
ts=2026-04-26T12:44:55.579955997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:50900"
ts=2026-04-26T12:45:07.478132678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47556"
ts=2026-04-26T12:45:07.486275915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47572"
ts=2026-04-26T12:45:10.597229538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:45:23.263964643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33714"
ts=2026-04-26T12:45:25.58144206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:60664"
ts=2026-04-26T12:45:40.598856342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:45:47.320228482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:45:47.321838337Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:45:53.265313137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59760"
ts=2026-04-26T12:46:09.438764736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35430"
ts=2026-04-26T12:46:10.600206411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:46:23.878658038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:46:39.44006084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:49982"
ts=2026-04-26T12:46:40.601244545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:47:09.441696597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:35078"
level=debug ts=2026-04-26T12:47:09.759421374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:47:10.602833029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:47:40.605230262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:47:53.27279481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49412"
ts=2026-04-26T12:47:55.588008309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:44918"
level=debug ts=2026-04-26T12:48:00.534982852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:48:10.606501954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
level=debug ts=2026-04-26T12:48:37.73582462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:48:39.445167807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:32816"
ts=2026-04-26T12:48:40.607447903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:48:53.275614557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59130"
ts=2026-04-26T12:48:55.591022903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:48366"
ts=2026-04-26T12:49:10.608465736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:49:26.856612507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:49:26.858238364Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:49:40.609958546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:49:55.594082211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:44908"
ts=2026-04-26T12:50:10.610988868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
level=debug ts=2026-04-26T12:50:12.822153486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:50:25.595133335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:41378"
ts=2026-04-26T12:50:40.612700154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
ts=2026-04-26T12:50:53.281382263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48102"
level=debug ts=2026-04-26T12:51:09.835906187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:51:10.614241568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
ts=2026-04-26T12:51:25.597310827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:42286"
ts=2026-04-26T12:51:40.615251888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:52:06.196968366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:52:09.454102885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34608"
ts=2026-04-26T12:52:10.616466735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:52:23.284181378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50922"
ts=2026-04-26T12:52:40.617746581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:52:56.784928872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:53:10.618885054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
ts=2026-04-26T12:53:23.287212058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51706"
level=error ts=2026-04-26T12:53:23.422130828Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:53:23.423977561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:53:23.425847573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-26T12:53:23.425873737Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:53:25.60293257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:48954"
ts=2026-04-26T12:53:40.619974273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
level=debug ts=2026-04-26T12:53:51.366096402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:53:51.367579543Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:53:53.288444882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40580"
ts=2026-04-26T12:54:10.620959215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-362c69f7 10.130.0.83:7946"
ts=2026-04-26T12:54:40.622737986Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-362c69f7 failed: dial tcp 10.130.0.83:7946: connect: connection refused"
ts=2026-04-26T12:54:44.241651622Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-362c69f7 (timeout reached)"
ts=2026-04-26T12:54:47.241258717Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-362c69f7 has failed, no acks received"
level=debug ts=2026-04-26T12:54:51.833931401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:55:09.475208387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42466"
ts=2026-04-26T12:55:09.483389544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42472"
ts=2026-04-26T12:55:10.6235238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:55:40.62521045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T12:55:41.849810806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:55:55.60754099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:40576"
ts=2026-04-26T12:56:10.625905183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574f4841 10.130.0.83:7946"
level=debug ts=2026-04-26T12:56:17.733172436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:56:40.626894942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:57:10.627998032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574f4841 10.130.0.83:7946"
level=debug ts=2026-04-26T12:57:11.749961116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:57:40.629327068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:57:52.373215111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:42638"
ts=2026-04-26T12:57:55.612562791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43104"
level=debug ts=2026-04-26T12:58:05.896063814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-26T12:58:05.897548003Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-26T12:58:10.630729896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T12:58:40.631498289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T12:58:53.300568209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52716"
level=debug ts=2026-04-26T12:58:59.545437888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:59:10.632711351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-574f4841 10.130.0.83:7946"
ts=2026-04-26T12:59:22.37662664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:40334"
level=debug ts=2026-04-26T12:59:35.551037992Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T12:59:40.633504015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"
ts=2026-04-26T13:00:10.634385057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T13:00:10.770551949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T13:00:22.379105788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:34654"
ts=2026-04-26T13:00:40.635936056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
level=debug ts=2026-04-26T13:01:00.499419504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-26T13:01:10.637371047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-dt5rv-cd4a9a9a 10.130.0.81:7946"
ts=2026-04-26T13:01:22.382370763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.83:47736"
ts=2026-04-26T13:01:40.638762619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9gmgv-800d9f0d 10.130.0.82:7946"