level=info ts=2026-04-20T12:13:55.161992604Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T12:13:55.162042814Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T12:13:55.163736338Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T12:13:55.164168625Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:13:55.164284366Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.87 inf=eth0
level=debug ts=2026-04-20T12:13:55.164765731Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:13:55.164829884Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.87 inf=eth0
level=info ts=2026-04-20T12:13:55.164922959Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-66cd61ac
ts=2026-04-20T12:13:55.164992496Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T12:13:55.165015279Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.87 advertisePort=7946
level=debug ts=2026-04-20T12:13:55.16503052Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.87 advertisePort=7946
level=info ts=2026-04-20T12:13:55.165165231Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T12:13:55.172046003Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:13:55.172063222Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:13:55.172075165Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.912863ms
level=error ts=2026-04-20T12:13:55.17208769Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T12:13:55.17209705Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T12:13:55.174742412Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:13:55.174761803Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:13:55.179373547Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-20T12:13:55.179388395Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-20T12:13:55.179395346Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-20T12:13:55.17939639Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T12:13:55.179409111Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T12:13:55.179449299Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T12:13:55.179500778Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T12:13:55.179523386Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-20T12:13:55.179715955Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T12:13:55.17973325Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T12:13:55.17978097Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T12:13:55.179782231Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T12:13:55.179798531Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T12:13:55.179934226Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T12:13:55.17994058Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T12:13:55.179950145Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T12:13:56.180601031Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T12:13:56.37114706Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T12:13:56.371199602Z caller=loki.go:599 msg="Loki started" startup_time=1.272677174s
ts=2026-04-20T12:13:56.633096348Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:13:56.633121171Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:13:59.986230969Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:13:59.986264995Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T12:14:01.372168806Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T12:14:01.372227262Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T12:14:06.795919778Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:14:06.795946079Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:14:09.796457104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:14:14.323726448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56612"
ts=2026-04-20T12:14:21.037664656Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
ts=2026-04-20T12:14:21.038789631Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
ts=2026-04-20T12:14:21.039580811Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.86:7946: dial tcp 10.128.0.86:7946: connect: connection refused"
ts=2026-04-20T12:14:21.039771549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.85:7946"
ts=2026-04-20T12:14:21.040335065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.87:7946"
ts=2026-04-20T12:14:21.040370544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43192"
level=info ts=2026-04-20T12:14:21.041001041Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=25.868893234s
ts=2026-04-20T12:14:21.076525242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37328"
ts=2026-04-20T12:14:24.693837753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34424"
level=debug ts=2026-04-20T12:14:27.593797553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:14:28.008533633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:44802"
ts=2026-04-20T12:14:29.52957656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:14:29.603490968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34466"
level=debug ts=2026-04-20T12:14:44.283068998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:14:59.531796217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:14:59.606123819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58990"
level=debug ts=2026-04-20T12:15:11.513639395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:15:28.013403639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41064"
ts=2026-04-20T12:15:29.534298106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:15:39.542071395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:15:39.544071184Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:15:50.587518961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45534"
ts=2026-04-20T12:15:59.537775277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:16:20.59053259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49308"
level=debug ts=2026-04-20T12:16:23.740517465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:16:29.540763223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:16:29.611428687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58248"
ts=2026-04-20T12:16:58.021504214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50366"
ts=2026-04-20T12:16:59.543655893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:17:29.546547757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:17:36.782573225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:17:50.600050961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36552"
ts=2026-04-20T12:17:58.025111578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46228"
ts=2026-04-20T12:17:59.549662438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:18:29.552705193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:18:29.620175722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34216"
level=debug ts=2026-04-20T12:18:49.175648089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:18:58.030197996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56658"
ts=2026-04-20T12:18:59.556125079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:18:59.623583095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33264"
ts=2026-04-20T12:19:28.031564257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48546"
ts=2026-04-20T12:19:29.557984397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:19:29.626882593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58744"
level=debug ts=2026-04-20T12:19:51.54405968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:19:58.032491015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58228"
ts=2026-04-20T12:19:59.558764658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:20:29.561276846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:20:50.609056421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59854"
ts=2026-04-20T12:20:59.564099006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:20:59.633602539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46330"
level=debug ts=2026-04-20T12:21:02.0803971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:21:02.082388179Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:21:29.566892809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:21:29.636472667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41374"
ts=2026-04-20T12:21:59.570174926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:22:01.163750337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:22:28.046289479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48720"
ts=2026-04-20T12:22:29.572749943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:22:29.641343213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43340"
level=debug ts=2026-04-20T12:22:47.024709156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:22:58.047761379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42514"
ts=2026-04-20T12:22:59.575700459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:23:29.578641215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:23:48.223298528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:23:58.052630181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45474"
ts=2026-04-20T12:23:59.581619952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=info ts=2026-04-20T12:24:01.373241037Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T12:24:01.378662577Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:24:01.381274245Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:24:01.383451727Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:24:01.383481491Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:24:01.383496794Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-20T12:24:25.479599639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:24:29.585610657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:24:58.057703728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58488"
ts=2026-04-20T12:24:59.588590701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:25:19.726443285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:25:19.72827412Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:25:29.592169692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:25:50.631893407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35990"
ts=2026-04-20T12:25:58.062581977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49894"
ts=2026-04-20T12:25:59.595405233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:26:20.634972044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48008"
level=debug ts=2026-04-20T12:26:21.664319481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:26:29.598729103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:26:50.638618666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44472"
ts=2026-04-20T12:26:59.600535527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:27:09.786806674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:27:20.642318187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50070"
ts=2026-04-20T12:27:28.069900877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39808"
ts=2026-04-20T12:27:29.602586164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:27:56.003622192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:27:59.605049092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:28:20.649211996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60238"
ts=2026-04-20T12:28:29.60641887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:28:29.672387125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48046"
level=debug ts=2026-04-20T12:28:42.592373738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:28:50.652196245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53994"
ts=2026-04-20T12:28:59.608349648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:29:28.080840273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58030"
level=debug ts=2026-04-20T12:29:28.218260478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:29:28.220259272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:29:29.611201959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:29:29.677983714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48224"
ts=2026-04-20T12:29:59.614864025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:29:59.680961338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48886"
level=debug ts=2026-04-20T12:30:12.797793951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:30:28.085351832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56574"
ts=2026-04-20T12:30:29.618258788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:30:29.684270339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53986"
ts=2026-04-20T12:30:59.621552027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
level=debug ts=2026-04-20T12:31:05.297636144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:31:29.624324001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:31:50.665941532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49096"
ts=2026-04-20T12:31:58.092730964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41368"
ts=2026-04-20T12:31:59.626955647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
level=debug ts=2026-04-20T12:32:06.520743214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:32:29.629616016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:32:29.694835546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46846"
level=debug ts=2026-04-20T12:32:46.648952597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:32:59.630619563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:33:26.512853624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:33:26.514436822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:33:29.631611261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:33:50.677148794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33094"
ts=2026-04-20T12:33:59.633572747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:33:59.703023416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35640"
level=error ts=2026-04-20T12:34:01.389112183Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:34:01.39114803Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:34:01.392888212Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:34:01.392919053Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:34:19.407174938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:34:28.106787225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48660"
ts=2026-04-20T12:34:29.63653769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:34:29.705709898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52484"
ts=2026-04-20T12:34:59.639382672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:35:08.894560285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:35:29.641425209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:35:58.11303011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:37046"
level=debug ts=2026-04-20T12:35:58.45115896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:35:59.642449651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:36:29.643826703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:36:29.713590964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57138"
level=debug ts=2026-04-20T12:36:37.391291562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:36:58.118566211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38384"
ts=2026-04-20T12:36:59.646307428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:37:20.893674859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:37:20.895258107Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:37:28.120221282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43702"
ts=2026-04-20T12:37:29.648901758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:37:29.71820921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54312"
level=debug ts=2026-04-20T12:37:56.572750565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:37:58.121469833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59246"
ts=2026-04-20T12:37:59.651432723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:38:29.654354314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:38:29.724350936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58008"
level=debug ts=2026-04-20T12:38:32.28416247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:38:59.656207014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:38:59.727994438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48896"
level=debug ts=2026-04-20T12:39:17.275502211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:39:28.129444714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43412"
ts=2026-04-20T12:39:29.657479009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:39:29.73109423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48420"
ts=2026-04-20T12:39:59.659396396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:39:59.734026346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38944"
level=debug ts=2026-04-20T12:40:08.590825605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:40:28.134293685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41882"
ts=2026-04-20T12:40:29.661637123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:40:29.736982777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43652"
level=debug ts=2026-04-20T12:40:52.915376991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:40:52.91721202Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:40:58.135453356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43984"
ts=2026-04-20T12:40:59.662707837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:40:59.740518911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37216"
ts=2026-04-20T12:41:29.665620666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:41:47.386241231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:41:50.718151161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39412"
ts=2026-04-20T12:41:59.669223985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:42:29.672068114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:42:37.006182342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:42:58.147390897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34246"
ts=2026-04-20T12:42:59.67469591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:43:20.72730332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57788"
level=debug ts=2026-04-20T12:43:22.497982707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:43:29.677323285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:43:29.755869526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40896"
ts=2026-04-20T12:43:50.730017606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34934"
ts=2026-04-20T12:43:59.67924594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:43:59.759273438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48874"
level=error ts=2026-04-20T12:44:01.387790939Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:44:01.389930704Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:44:01.391714114Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:44:01.391740764Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:44:10.865196616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:44:29.681970998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:44:29.762685699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49446"
ts=2026-04-20T12:44:50.736940268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36270"
ts=2026-04-20T12:44:59.685755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
level=debug ts=2026-04-20T12:45:00.055557104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:45:00.057335224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:45:29.689099008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:45:29.767743776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42776"
level=debug ts=2026-04-20T12:45:55.63376691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:45:59.69116309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:46:29.692871098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:46:29.952319476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:46:59.693857229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:47:19.828978098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:47:20.749488525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51100"
ts=2026-04-20T12:47:29.696800164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:47:29.777527051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51318"
ts=2026-04-20T12:47:50.752842844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58138"
ts=2026-04-20T12:47:58.177680619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45446"
ts=2026-04-20T12:47:59.699413587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:48:19.411719303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:48:28.178969623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55760"
ts=2026-04-20T12:48:29.70518187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:48:58.180675646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38146"
ts=2026-04-20T12:48:59.709056673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:48:59.786856269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41796"
level=debug ts=2026-04-20T12:49:15.940039037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:49:15.941754519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:49:29.712725514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:49:29.789858377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54302"
ts=2026-04-20T12:49:50.768894794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45604"
ts=2026-04-20T12:49:59.715043777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:50:10.165504951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:50:29.717035762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:50:29.793940165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58248"
ts=2026-04-20T12:50:59.720586723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:51:08.637000733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:51:20.77667104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53994"
ts=2026-04-20T12:51:29.722273983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:51:50.779629685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60138"
ts=2026-04-20T12:51:59.723614289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=debug ts=2026-04-20T12:52:05.99971312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:52:29.726287469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:52:29.804069644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46334"
ts=2026-04-20T12:52:58.20373068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49680"
ts=2026-04-20T12:52:59.728415541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:52:59.806954926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38450"
level=debug ts=2026-04-20T12:53:00.208316415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:53:29.731399734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:53:29.809952081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52862"
ts=2026-04-20T12:53:58.207866942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58012"
level=debug ts=2026-04-20T12:53:59.017985932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:53:59.01949644Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:53:59.733609277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
level=error ts=2026-04-20T12:54:01.386097736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:54:01.387664291Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:54:01.389144093Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:54:01.389169561Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:54:29.735764746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:54:58.213047475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:35934"
level=debug ts=2026-04-20T12:54:59.338972562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:54:59.738345493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:55:29.740488281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:55:29.817214884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46746"
level=debug ts=2026-04-20T12:55:49.458112617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:55:59.744291973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
ts=2026-04-20T12:56:20.80102467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55968"
ts=2026-04-20T12:56:29.747154581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
level=debug ts=2026-04-20T12:56:44.950369504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:56:50.803480632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53570"
ts=2026-04-20T12:56:59.750840568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:57:25.895038835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:57:29.75335621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:57:50.808570454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55430"
ts=2026-04-20T12:57:59.755218584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2217e324 10.130.0.72:7946"
ts=2026-04-20T12:57:59.832814236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53788"
level=debug ts=2026-04-20T12:58:14.228427562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:58:14.230377049Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:58:28.232944459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38068"
ts=2026-04-20T12:58:29.758200754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:58:57.094989535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:58:59.760299603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-26kgq-d7816f32 10.128.0.85:7946"
ts=2026-04-20T12:59:29.762311775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"
level=debug ts=2026-04-20T12:59:32.549212829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:59:50.818568977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49268"
ts=2026-04-20T12:59:59.76523323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dvx44-334a7db9 10.130.0.68:7946"