level=info ts=2026-04-30T09:53:39.41282614Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-30T09:53:39.412845881Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-30T09:53:39.414697894Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-30T09:53:39.415048858Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T09:53:39.41513256Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
level=debug ts=2026-04-30T09:53:39.415645255Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T09:53:39.415712154Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
level=info ts=2026-04-30T09:53:39.415726057Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-6b0cec1a
ts=2026-04-30T09:53:39.415789324Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-30T09:53:39.415804402Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=debug ts=2026-04-30T09:53:39.415829462Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=info ts=2026-04-30T09:53:39.415971727Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-30T09:53:39.421147752Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T09:53:39.421168903Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-30T09:53:39.421181997Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.212392ms
level=error ts=2026-04-30T09:53:39.421193867Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-30T09:53:39.421209745Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-30T09:53:39.425550485Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T09:53:39.425567569Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-30T09:53:39.43353507Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-30T09:53:39.43357123Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-30T09:53:39.433559585Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-30T09:53:39.433573798Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-30T09:53:39.433583702Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-30T09:53:39.433611373Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-30T09:53:39.43363765Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-30T09:53:39.433807895Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-30T09:53:39.433818842Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-30T09:53:39.433824574Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-30T09:53:39.433863644Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-30T09:53:39.433879199Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-30T09:53:39.43391008Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-30T09:53:39.434085005Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-30T09:53:39.434101575Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-30T09:53:39.434132771Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-30T09:53:40.434688492Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-30T09:53:40.624039479Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-30T09:53:40.624101355Z caller=loki.go:615 msg="Loki started" startup_time=1.275528883s
ts=2026-04-30T09:53:41.383597219Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T09:53:41.383622135Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:53:43.529777417Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T09:53:43.529799577Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-30T09:53:45.625139304Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-30T09:53:45.625182774Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-30T09:53:50.450365587Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T09:53:50.45038845Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-30T09:53:54.724854431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:54:00.934145236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54138"
ts=2026-04-30T09:54:02.052941475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33108"
ts=2026-04-30T09:54:05.606897054Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
ts=2026-04-30T09:54:05.607970392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.72:7946"
ts=2026-04-30T09:54:05.610264621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.71:7946"
ts=2026-04-30T09:54:05.611673164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
ts=2026-04-30T09:54:05.611714657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40608"
ts=2026-04-30T09:54:05.612859753Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.85:7946: dial tcp 10.128.0.85:7946: connect: connection refused"
ts=2026-04-30T09:54:05.613467498Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.81:7946: dial tcp 10.128.0.81:7946: connect: connection refused"
level=info ts=2026-04-30T09:54:05.613479316Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.192262245s
ts=2026-04-30T09:54:06.559810941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53480"
level=debug ts=2026-04-30T09:54:13.098781635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:54:25.320146646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43536"
ts=2026-04-30T09:54:27.510567839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46216"
level=debug ts=2026-04-30T09:54:30.514181172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:54:31.553084525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T09:54:43.115306841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51098"
level=debug ts=2026-04-30T09:54:50.542108723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:54:57.514057694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43504"
ts=2026-04-30T09:55:01.554536072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T09:55:24.725857655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T09:55:24.72762889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T09:55:25.325107766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52338"
ts=2026-04-30T09:55:31.555862166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T09:55:43.119784336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51946"
level=debug ts=2026-04-30T09:55:57.428150979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:55:57.520461495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33934"
ts=2026-04-30T09:56:01.557984017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T09:56:13.122612925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55892"
ts=2026-04-30T09:56:31.561444635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T09:56:55.902843964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:57:01.564594785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T09:57:25.335041503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49040"
ts=2026-04-30T09:57:31.567615997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T09:57:43.128723328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59098"
ts=2026-04-30T09:58:01.570442994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T09:58:09.623270312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:58:13.131408897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34654"
ts=2026-04-30T09:58:31.573034837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T09:58:55.343129927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44202"
ts=2026-04-30T09:58:57.535039377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40856"
ts=2026-04-30T09:59:01.57602286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T09:59:18.373056375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T09:59:31.578931848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T09:59:57.541069217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45040"
ts=2026-04-30T10:00:01.581649385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:00:12.181653278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T10:00:12.183177938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:00:25.350278279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58568"
ts=2026-04-30T10:00:27.54413615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58432"
ts=2026-04-30T10:00:31.584296167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:00:50.238998772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:00:55.352029526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38100"
ts=2026-04-30T10:00:57.547120522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50444"
ts=2026-04-30T10:01:01.5872168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:01:13.143720614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35400"
level=debug ts=2026-04-30T10:01:29.944101893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:01:31.589617086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:02:01.591805463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:02:06.246450275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:02:13.149914767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46078"
ts=2026-04-30T10:02:25.360556043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36956"
ts=2026-04-30T10:02:31.595191377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:02:55.361933958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38500"
ts=2026-04-30T10:02:57.555864346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54016"
ts=2026-04-30T10:03:01.598764793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:03:05.298559467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:03:25.363512754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52582"
ts=2026-04-30T10:03:27.558395043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44638"
ts=2026-04-30T10:03:31.600793533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:03:43.157956258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59812"
level=info ts=2026-04-30T10:03:45.626174621Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-30T10:03:45.629988156Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:03:45.631944636Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:03:45.63391647Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:03:45.633943923Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T10:03:45.633956063Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-30T10:03:55.364399746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36272"
level=debug ts=2026-04-30T10:04:00.019122956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:04:00.020648705Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:04:01.602534122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:04:13.160175337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42378"
ts=2026-04-30T10:04:25.365769309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58560"
ts=2026-04-30T10:04:27.562868389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42984"
ts=2026-04-30T10:04:31.605925892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:04:44.94854386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:04:55.367238168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38012"
ts=2026-04-30T10:04:57.56550743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43152"
ts=2026-04-30T10:05:01.608791879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:05:23.627714962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:05:31.611483401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:05:43.167731187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38260"
ts=2026-04-30T10:06:01.612912447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:06:06.29686876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:06:13.171244696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46284"
ts=2026-04-30T10:06:25.374746054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56692"
ts=2026-04-30T10:06:31.614644591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:06:49.899098078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:06:57.574506516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50852"
ts=2026-04-30T10:07:01.618248743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:07:30.879390693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:07:30.88128516Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:07:31.621602309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:07:43.181436171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59330"
ts=2026-04-30T10:07:55.383348922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39254"
ts=2026-04-30T10:08:01.62560838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:08:09.042888686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:08:27.580397078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58988"
ts=2026-04-30T10:08:31.627716056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:08:55.387407165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46724"
ts=2026-04-30T10:09:01.629939503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:09:03.065610041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:09:13.191648898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41526"
ts=2026-04-30T10:09:31.632996294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:09:38.10287648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:09:57.58609333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56612"
ts=2026-04-30T10:10:01.635104838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:10:16.506771606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:10:27.589061594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50840"
ts=2026-04-30T10:10:31.637457194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:10:43.198925441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42748"
ts=2026-04-30T10:11:01.640298048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:11:08.403455639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:11:08.405456938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:11:31.642338576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:11:50.177984015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:12:01.643615946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:12:25.408102454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45194"
ts=2026-04-30T10:12:31.646280676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:12:33.353021304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:12:43.209421918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49924"
ts=2026-04-30T10:12:57.599968075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50806"
ts=2026-04-30T10:13:01.648647553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:13:08.179660592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:13:25.41210065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37646"
ts=2026-04-30T10:13:27.6034369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58200"
ts=2026-04-30T10:13:31.651361329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:13:43.215668814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41684"
level=error ts=2026-04-30T10:13:45.638301535Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:13:45.640150338Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:13:45.642148167Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:13:45.642172275Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-30T10:13:51.39448834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:13:57.606618254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45400"
ts=2026-04-30T10:14:01.654847371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:14:31.657931636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:14:34.863440928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:14:34.865404645Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:14:55.420044221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40726"
ts=2026-04-30T10:15:01.66002203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:15:22.353581446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:15:31.661812197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:15:43.225870233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36376"
ts=2026-04-30T10:15:57.620531641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46254"
level=debug ts=2026-04-30T10:15:59.966499523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:16:01.663555864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:16:31.665874932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:16:53.097760364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:17:01.668017978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:17:31.668869814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:17:34.468735327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:17:43.233890544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54808"
ts=2026-04-30T10:18:01.670608062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:18:27.632151536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45154"
ts=2026-04-30T10:18:31.672781411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:18:35.252023159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:18:35.254041436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:18:57.635358288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50058"
ts=2026-04-30T10:19:01.673924055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:19:13.243500712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36216"
ts=2026-04-30T10:19:25.445760517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34996"
level=debug ts=2026-04-30T10:19:31.179722569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:19:31.676475701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:20:01.679565152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:20:07.349143017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:20:31.682026888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:21:01.684631631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:21:01.939919424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:21:31.686366388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:21:52.413786458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:22:01.688186593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:22:31.6898065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:22:48.359915024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:22:48.361745902Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:22:55.465280068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36578"
ts=2026-04-30T10:23:01.691892114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:23:31.694645736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:23:32.121968449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:23:45.637831224Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:23:45.639932841Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:23:45.642010113Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:23:45.642031153Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:24:01.697094102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:24:13.261818303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37668"
ts=2026-04-30T10:24:31.699831451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:24:32.825991526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:24:55.476555827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50998"
ts=2026-04-30T10:24:57.666466086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54790"
ts=2026-04-30T10:25:01.705373581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:25:31.708691848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:25:32.207848779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:25:57.670189101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36462"
ts=2026-04-30T10:26:01.716152574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:26:25.484598417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50202"
level=debug ts=2026-04-30T10:26:26.900639194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:26:27.673432292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:46320"
ts=2026-04-30T10:26:31.71906145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:27:01.722280949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:27:15.503412763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:27:15.504901965Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:27:25.489082462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:53076"
ts=2026-04-30T10:27:31.725592675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:27:52.736483909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:27:57.681341193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42188"
ts=2026-04-30T10:28:01.728491529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:28:13.281193189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33186"
ts=2026-04-30T10:28:25.493337578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39884"
ts=2026-04-30T10:28:27.684711667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43448"
level=debug ts=2026-04-30T10:28:29.887228791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:28:31.731433229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:28:57.687482366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59216"
ts=2026-04-30T10:29:01.734468745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:29:24.251269565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:29:25.497706295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46692"
ts=2026-04-30T10:29:27.690938153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45280"
ts=2026-04-30T10:29:31.736712591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:30:01.73875639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:30:13.290073697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52026"
level=debug ts=2026-04-30T10:30:20.958163027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:30:25.501524772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43476"
ts=2026-04-30T10:30:27.696806913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59960"
ts=2026-04-30T10:30:31.742104894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:31:01.744510065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:31:05.829616776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:31:05.831415484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:31:13.294120104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50346"
ts=2026-04-30T10:31:31.746414413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:31:46.1955447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:31:57.703698659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:55156"
ts=2026-04-30T10:32:01.748988758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:32:31.752407383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:32:35.177929564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:32:43.301409672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54980"
ts=2026-04-30T10:33:01.755185322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:33:22.561558511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:33:25.517422789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36162"
ts=2026-04-30T10:33:31.758047845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=error ts=2026-04-30T10:33:45.63786124Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:33:45.639781063Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:33:45.641392833Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:33:45.64141404Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:34:01.761540636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:34:12.711977308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:34:13.307180711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49610"
ts=2026-04-30T10:34:27.719684252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42672"
ts=2026-04-30T10:34:31.763367358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:34:43.310280008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46342"
ts=2026-04-30T10:34:55.525381003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41682"
ts=2026-04-30T10:35:01.765838829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:35:13.73953053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:35:13.741257629Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:35:25.526747477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35430"
ts=2026-04-30T10:35:31.767570554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:35:43.314843234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46486"
ts=2026-04-30T10:36:01.770446584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:36:05.944268454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:36:13.31752418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57920"
ts=2026-04-30T10:36:31.775605067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:36:53.182212935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:37:01.777798477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:37:27.731953773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60490"
ts=2026-04-30T10:37:31.781092311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:37:33.137838372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:37:43.325611984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43946"
ts=2026-04-30T10:38:01.783946103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:38:27.737382791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39748"
level=debug ts=2026-04-30T10:38:29.875359079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:38:31.787460023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:38:57.740271187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59198"
ts=2026-04-30T10:39:01.790298474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:39:08.903958228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:39:08.905708963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:39:31.793775819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:39:57.173729324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:40:01.79564076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:40:31.796868982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:40:43.340359824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59690"
level=debug ts=2026-04-30T10:40:50.952095959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:40:55.559035348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:53600"
ts=2026-04-30T10:41:01.799646974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:41:25.560520761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39146"
ts=2026-04-30T10:41:31.802651713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:41:32.380845133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:41:43.345964804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59552"
ts=2026-04-30T10:42:01.805550905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:42:07.982913377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:42:13.349359897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35250"
ts=2026-04-30T10:42:31.80901157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:42:43.352291004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59514"
level=debug ts=2026-04-30T10:42:51.567779542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:42:51.569331686Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:42:57.756600303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39578"
ts=2026-04-30T10:43:01.810885806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:43:31.812971989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:43:43.357302371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39984"
level=error ts=2026-04-30T10:43:45.637408379Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:43:45.639299124Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:43:45.641152863Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:43:45.641176588Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-30T10:43:46.33778348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:43:55.574814695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48788"
ts=2026-04-30T10:44:01.816125005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:44:13.360323748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58044"
ts=2026-04-30T10:44:31.818636065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:44:37.207021876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:44:55.578983062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58776"
ts=2026-04-30T10:45:01.819894123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:45:13.364616675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39586"
ts=2026-04-30T10:45:31.821545956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:45:37.050064631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:45:43.36786101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51158"
ts=2026-04-30T10:45:55.583351886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38098"
ts=2026-04-30T10:45:57.769793907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42536"
ts=2026-04-30T10:46:01.823881455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:46:13.370701484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43274"
ts=2026-04-30T10:46:31.826690081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:46:39.107060804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:46:55.588248123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38768"
ts=2026-04-30T10:47:01.828837342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:47:13.375331297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41574"
level=debug ts=2026-04-30T10:47:24.308023404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:47:24.30970308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:47:25.589403017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36016"
ts=2026-04-30T10:47:31.83092363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:47:43.378372283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39018"
ts=2026-04-30T10:47:57.779369211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42486"
ts=2026-04-30T10:48:01.833010047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:48:08.928640145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:48:13.381355136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36836"
ts=2026-04-30T10:48:31.835463675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:48:56.196418387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:49:01.838835219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:49:31.842169289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:49:50.200575897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:50:01.844794355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:50:13.393203338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36776"
ts=2026-04-30T10:50:25.60561921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58410"
ts=2026-04-30T10:50:27.791785321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41128"
ts=2026-04-30T10:50:31.847025029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:50:47.752228752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:50:55.607112576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46826"
ts=2026-04-30T10:51:01.848925373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:51:25.608566517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56524"
ts=2026-04-30T10:51:31.851692659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:51:41.992613187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:51:41.994343079Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:51:43.400058296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41046"
ts=2026-04-30T10:52:01.853712268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
level=debug ts=2026-04-30T10:52:18.743986116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:52:25.612751513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33718"
ts=2026-04-30T10:52:31.855680192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:52:55.614009042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:32906"
ts=2026-04-30T10:53:01.857491996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:53:13.408886972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42856"
level=debug ts=2026-04-30T10:53:19.760844818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:53:31.859155659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
ts=2026-04-30T10:53:43.411668957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54932"
level=error ts=2026-04-30T10:53:45.637996969Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:53:45.639646759Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:53:45.641204162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T10:53:45.64123058Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:53:57.805939758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36468"
ts=2026-04-30T10:54:01.861974961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:54:11.843442174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:54:25.621964936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38778"
ts=2026-04-30T10:54:31.86496315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:54:57.812597006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50306"
ts=2026-04-30T10:55:01.867455121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-26zr9-a194efce 10.129.0.72:7946"
level=debug ts=2026-04-30T10:55:06.654998722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:55:25.625560581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40832"
ts=2026-04-30T10:55:31.870191655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
level=debug ts=2026-04-30T10:55:47.039290333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T10:55:47.040978019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T10:55:55.627088361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52008"
ts=2026-04-30T10:56:01.87346138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"
ts=2026-04-30T10:56:25.628564303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39584"
level=debug ts=2026-04-30T10:56:26.388480177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T10:56:27.819880548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37050"
ts=2026-04-30T10:56:31.875058842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51e848be 10.128.0.83:7946"
ts=2026-04-30T10:56:55.62984895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45358"
ts=2026-04-30T10:56:57.823170382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34046"
ts=2026-04-30T10:57:01.877369305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-s8xkm-17684b53 10.129.0.71:7946"