level=info ts=2026-04-22T19:00:09.067980345Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T19:00:09.068018289Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T19:00:09.069391291Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T19:00:09.069784858Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T19:00:09.069884193Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.78 inf=eth0
level=debug ts=2026-04-22T19:00:09.071043193Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T19:00:09.071130273Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.78 inf=eth0
level=info ts=2026-04-22T19:00:09.071161081Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-abf4722a
ts=2026-04-22T19:00:09.071230068Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T19:00:09.071255879Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.78 advertisePort=7946
level=debug ts=2026-04-22T19:00:09.071273637Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.78 advertisePort=7946
level=info ts=2026-04-22T19:00:09.071405109Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T19:00:09.076694436Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:00:09.076713884Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T19:00:09.076725288Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.322188ms
level=error ts=2026-04-22T19:00:09.076733981Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T19:00:09.076742766Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T19:00:09.081102127Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:00:09.081123041Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T19:00:09.087800709Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T19:00:09.087947779Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T19:00:09.087966238Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T19:00:09.087984205Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T19:00:09.088079671Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T19:00:09.088103236Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-22T19:00:09.088752818Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-22T19:00:09.088766913Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T19:00:09.088773294Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-22T19:00:09.088778187Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T19:00:09.089090044Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T19:00:09.089144321Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T19:00:09.089166942Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T19:00:09.089337246Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T19:00:09.089345489Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T19:00:09.089359943Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T19:00:10.089634043Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T19:00:10.281537033Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T19:00:10.281580992Z caller=loki.go:599 msg="Loki started" startup_time=1.289827758s
ts=2026-04-22T19:00:11.052688446Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:00:11.052715149Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:14.133428491Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:00:14.133452558Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T19:00:15.282566217Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T19:00:15.282628213Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T19:00:19.359688671Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:00:19.359713217Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T19:00:20.644465463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:30.894107319Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:00:30.894130768Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:32.360606427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35286"
ts=2026-04-22T19:00:36.50288303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35074"
level=debug ts=2026-04-22T19:00:44.023826994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:44.495691136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47958"
ts=2026-04-22T19:00:46.081135419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51968"
ts=2026-04-22T19:00:50.990199712Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.75:7946: dial tcp 10.128.0.75:7946: connect: connection refused"
ts=2026-04-22T19:00:50.990390718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.76:7946"
ts=2026-04-22T19:00:50.991114597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.78:7946"
ts=2026-04-22T19:00:50.99114125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:44760"
ts=2026-04-22T19:00:50.99173465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-04-22T19:00:50.992500617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-04-22T19:00:50.9938789Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.73:7946: dial tcp 10.128.0.73:7946: connect: connection refused"
ts=2026-04-22T19:00:50.994632685Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.77:7946: dial tcp 10.128.0.77:7946: connect: connection refused"
level=info ts=2026-04-22T19:00:50.994647476Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=41.917899359s
ts=2026-04-22T19:01:01.68588645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40892"
ts=2026-04-22T19:01:04.109920362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56004"
ts=2026-04-22T19:01:05.761976728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:01:06.692998523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T19:01:31.126640996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:01:31.687655777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54790"
ts=2026-04-22T19:01:35.763332897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:01:59.234452118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:01:59.236114777Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:02:05.765064129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:02:34.11383031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38738"
ts=2026-04-22T19:02:35.766244769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:02:39.744365486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:03:01.691588437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48692"
ts=2026-04-22T19:03:05.767693128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
ts=2026-04-22T19:03:16.086945262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56568"
ts=2026-04-22T19:03:34.117109773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55848"
ts=2026-04-22T19:03:35.768872123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:03:38.118576868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:04:01.693609372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44674"
ts=2026-04-22T19:04:05.769920128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:04:34.120115294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45642"
ts=2026-04-22T19:04:35.771434318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:04:48.964763055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:05:01.696641667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45890"
ts=2026-04-22T19:05:04.12095733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57918"
ts=2026-04-22T19:05:05.772777234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:05:35.77425472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:05:52.877149901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:06:05.775931339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:06:35.777412245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:06:43.835602208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:06:43.837224127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:07:05.778954513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:07:16.099213033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54224"
level=debug ts=2026-04-22T19:07:29.090538086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:07:31.70212339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52072"
ts=2026-04-22T19:07:35.77997033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:08:04.128961074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57236"
ts=2026-04-22T19:08:05.781390715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:08:19.475974897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:08:35.782336941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:08:57.245992673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:09:01.707149062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58606"
ts=2026-04-22T19:09:05.783236955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:09:16.104525523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47080"
ts=2026-04-22T19:09:34.132912061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:50372"
level=debug ts=2026-04-22T19:09:35.172108014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:09:35.784129951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:10:05.785650648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=info ts=2026-04-22T19:10:15.282998869Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T19:10:15.286527256Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:10:15.288266009Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:10:15.290044556Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:10:15.290074308Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:10:15.290086885Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-22T19:10:27.310303173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:10:27.312147195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:10:31.710647819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33716"
ts=2026-04-22T19:10:35.786418375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:11:05.787535666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:11:06.862669837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:11:16.109493858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47010"
ts=2026-04-22T19:11:35.789206798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:12:01.414216816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:05.790773313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:12:16.112277901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39680"
ts=2026-04-22T19:12:35.791626178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:12:45.871971361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:46.113797047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60092"
ts=2026-04-22T19:13:04.141453058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40452"
ts=2026-04-22T19:13:05.793178727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
ts=2026-04-22T19:13:16.115493409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58160"
ts=2026-04-22T19:13:35.793936593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:13:45.280034525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:14:01.720758118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37584"
ts=2026-04-22T19:14:04.144683127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41284"
ts=2026-04-22T19:14:05.795044454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:14:31.722631197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35408"
level=debug ts=2026-04-22T19:14:34.821485047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:14:34.823178342Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:14:35.796288223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
ts=2026-04-22T19:14:46.119683438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53060"
ts=2026-04-22T19:15:05.797508603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:15:20.360311783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:15:35.799445175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:16:04.110573361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:16:04.150347381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55048"
ts=2026-04-22T19:16:05.800729405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:16:35.802414278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:16:39.458936899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:16:46.12569346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49070"
ts=2026-04-22T19:17:01.729095188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41308"
ts=2026-04-22T19:17:04.153044283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:35976"
ts=2026-04-22T19:17:05.803494519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:17:33.582264488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:17:34.154294854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:44386"
ts=2026-04-22T19:17:35.804945604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:17:46.128457582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39258"
ts=2026-04-22T19:18:01.732660775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53286"
ts=2026-04-22T19:18:05.806412038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:18:24.85679295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:18:24.858688221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:18:31.734179483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48692"
ts=2026-04-22T19:18:34.157775317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42892"
ts=2026-04-22T19:18:35.808526239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:18:46.131593909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50008"
ts=2026-04-22T19:19:05.809872404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:19:16.132941047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44998"
level=debug ts=2026-04-22T19:19:24.587777222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:19:31.73715894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50202"
ts=2026-04-22T19:19:35.811339399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:19:46.134409673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51894"
ts=2026-04-22T19:20:05.812689822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=error ts=2026-04-22T19:20:15.294168057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:20:15.296854583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:20:15.298803109Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:20:15.298838031Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:20:16.13620422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38502"
level=debug ts=2026-04-22T19:20:18.375771197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:20:35.813705517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
ts=2026-04-22T19:21:01.741753552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36270"
ts=2026-04-22T19:21:04.16551703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42124"
ts=2026-04-22T19:21:05.814986694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:21:13.625764169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:21:35.816625498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:21:55.306665933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:22:05.817605517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:22:31.746486731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36478"
ts=2026-04-22T19:22:35.819425211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:22:46.14348171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43950"
level=debug ts=2026-04-22T19:22:49.073869731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:22:49.075732921Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:23:04.17018866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45300"
ts=2026-04-22T19:23:05.821228772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:23:32.712095559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:23:35.822367684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:24:01.75153719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:32902"
ts=2026-04-22T19:24:05.823370089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:24:09.766084576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:24:34.174403117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53470"
ts=2026-04-22T19:24:35.824453343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:25:01.755306001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44128"
ts=2026-04-22T19:25:04.17602165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:59852"
ts=2026-04-22T19:25:05.826108769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:25:05.83623131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:25:16.149343247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55234"
ts=2026-04-22T19:25:35.827022946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:25:57.695848522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:26:05.828776786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:26:35.831276919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7d549a37 10.128.0.79:7946"
level=debug ts=2026-04-22T19:26:57.21273722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:26:57.214508622Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:27:01.761463775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37448"
ts=2026-04-22T19:27:04.181713721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57746"
ts=2026-04-22T19:27:05.832972078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:27:06.872956121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55154"
ts=2026-04-22T19:27:06.87823063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55156"
ts=2026-04-22T19:27:31.071756861Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7d549a37 (timeout reached)"
ts=2026-04-22T19:27:34.072485009Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7d549a37 has failed, no acks received"
ts=2026-04-22T19:27:34.090621429Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7d549a37 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:27:35.834374006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:27:36.205373986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:27:43.920859082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48094"
ts=2026-04-22T19:28:05.836229902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:28:19.400064184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:28:35.838055871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:29:05.83928283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
level=debug ts=2026-04-22T19:29:18.769639303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:29:35.840324973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:29:43.926337914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49400"
ts=2026-04-22T19:30:05.841883772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:30:08.648241826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:30:13.927528989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56186"
level=error ts=2026-04-22T19:30:15.293624015Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:30:15.295929387Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:30:15.297697959Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:30:15.297726214Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:30:35.843050763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
ts=2026-04-22T19:30:43.928398673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43348"
level=debug ts=2026-04-22T19:30:48.820524814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:30:48.82264698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:31:01.771613644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44846"
ts=2026-04-22T19:31:05.845562048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
ts=2026-04-22T19:31:34.197259772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52274"
ts=2026-04-22T19:31:35.847424321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
level=debug ts=2026-04-22T19:31:47.606716331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:32:01.77430229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44688"
ts=2026-04-22T19:32:05.84910655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:32:13.932737481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54700"
ts=2026-04-22T19:32:34.200611454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:49720"
ts=2026-04-22T19:32:35.850774777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:32:40.805016686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:33:05.85222273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:33:30.660167111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:33:35.853336738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
ts=2026-04-22T19:33:43.937468659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42336"
ts=2026-04-22T19:34:01.779711014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57970"
ts=2026-04-22T19:34:05.854484479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6702e395 10.128.0.79:7946"
ts=2026-04-22T19:34:16.073952385Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6702e395 (timeout reached)"
ts=2026-04-22T19:34:19.072487388Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6702e395 has failed, no acks received"
level=debug ts=2026-04-22T19:34:31.309650982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:34:31.781544313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48686"
ts=2026-04-22T19:34:32.85558857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45438"
ts=2026-04-22T19:34:32.863529967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45450"
ts=2026-04-22T19:34:35.855569496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:34:39.073360647Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-6702e395 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:34:41.072529618Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6702e395 (timeout reached)"
ts=2026-04-22T19:34:44.071972834Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6702e395 has failed, no acks received"
ts=2026-04-22T19:35:05.433350809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39798"
ts=2026-04-22T19:35:05.856600511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:35:05.941342017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:35:05.943133174Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:35:34.210255758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54254"
ts=2026-04-22T19:35:35.858405721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:35:49.736711964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:36:04.211800396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43858"
ts=2026-04-22T19:36:05.435746733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50170"
ts=2026-04-22T19:36:05.859298471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:36:31.787817957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60914"
ts=2026-04-22T19:36:35.436655632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42888"
ts=2026-04-22T19:36:35.860409745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:36:43.019706431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:37:05.861873435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:37:23.169055212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:37:35.86352897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:38:05.440073826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34204"
ts=2026-04-22T19:38:05.864713456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-30c689be 10.128.0.79:7946"
level=debug ts=2026-04-22T19:38:10.173696456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:38:35.866390377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-30c689be 10.128.0.79:7946"
level=debug ts=2026-04-22T19:39:04.832192666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:39:04.833690626Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:39:05.86777431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-30c689be 10.128.0.79:7946"
ts=2026-04-22T19:39:31.796660316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40160"
ts=2026-04-22T19:39:35.868628841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:39:45.532112322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:40:01.798480638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47676"
ts=2026-04-22T19:40:04.2246976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56030"
ts=2026-04-22T19:40:05.870173139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=error ts=2026-04-22T19:40:15.293781512Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:40:15.296000454Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:40:15.297807017Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:40:15.297833742Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T19:40:26.109971075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:40:35.447761175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52026"
ts=2026-04-22T19:40:35.871534513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:41:01.286985017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:41:05.873366311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:41:35.874451197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:42:01.802770686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42418"
level=debug ts=2026-04-22T19:42:03.372355104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:42:05.875605605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-30c689be 10.128.0.79:7946"
ts=2026-04-22T19:42:35.876771121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:42:46.073079474Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-30c689be (timeout reached)"
ts=2026-04-22T19:42:49.072626025Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-30c689be has failed, no acks received"
ts=2026-04-22T19:42:56.073017827Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-30c689be (timeout reached)"
ts=2026-04-22T19:42:59.072169401Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-30c689be has failed, no acks received"
level=debug ts=2026-04-22T19:43:01.310423923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:43:01.312137502Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:43:03.863761416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52028"
ts=2026-04-22T19:43:03.869466313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52044"
ts=2026-04-22T19:43:04.233370484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43954"
ts=2026-04-22T19:43:05.878441668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:43:06.072550558Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-30c689be (timeout reached)"
ts=2026-04-22T19:43:09.072853063Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-30c689be has failed, no acks received"
ts=2026-04-22T19:43:09.072883535Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-30c689be as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:43:31.807326353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60578"
ts=2026-04-22T19:43:34.234218541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39052"
ts=2026-04-22T19:43:35.879846185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T19:43:50.343233028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:44:01.809068718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34876"
ts=2026-04-22T19:44:05.881043262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:44:06.22116103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53552"
level=debug ts=2026-04-22T19:44:26.989413395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:44:34.237519734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:43012"
ts=2026-04-22T19:44:35.882607863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:45:01.811837316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53660"
ts=2026-04-22T19:45:05.884137767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:45:06.065861055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:45:06.223142351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58700"
ts=2026-04-22T19:45:31.813361388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33836"
ts=2026-04-22T19:45:34.239922462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41704"
ts=2026-04-22T19:45:35.885793483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:45:54.371146864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:46:01.814855892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36416"
ts=2026-04-22T19:46:04.240755294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:57014"
ts=2026-04-22T19:46:05.886553956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:46:31.815735955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38318"
ts=2026-04-22T19:46:35.887747792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T19:46:52.016329283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:46:52.017833157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:47:01.817386045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59556"
ts=2026-04-22T19:47:05.888532912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:47:06.228870278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34782"
ts=2026-04-22T19:47:31.818313125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55702"
ts=2026-04-22T19:47:34.244144662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56130"
ts=2026-04-22T19:47:35.890003752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:47:48.728686448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:05.891136425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:48:25.80762353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:34.246367012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:56058"
ts=2026-04-22T19:48:35.892861044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:49:01.822556368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44726"
level=debug ts=2026-04-22T19:49:02.526725735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:49:05.893716895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:49:35.894769879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:49:48.775044209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:50:01.825281879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41518"
ts=2026-04-22T19:50:05.896096418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=error ts=2026-04-22T19:50:15.293939989Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:50:15.295605271Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:50:15.297194236Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:50:15.297219306Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:50:31.826998573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44790"
level=debug ts=2026-04-22T19:50:35.104029812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:50:35.105648172Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:50:35.897107358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:51:05.89815039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:51:15.306195461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:51:34.254475694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60408"
ts=2026-04-22T19:51:35.89922722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:52:03.540283177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:52:05.900101193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:52:31.833562481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49148"
ts=2026-04-22T19:52:35.901574418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T19:52:44.130466318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:53:01.834799829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38178"
ts=2026-04-22T19:53:05.903083472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:53:06.246694806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41170"
ts=2026-04-22T19:53:35.9041152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T19:53:46.167017935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:01.837193781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49738"
ts=2026-04-22T19:54:05.905828283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:54:06.249018572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57522"
ts=2026-04-22T19:54:34.26202907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42256"
ts=2026-04-22T19:54:35.907127245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:54:36.250002786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42784"
level=debug ts=2026-04-22T19:54:44.171960267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:54:44.173478827Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:55:05.908477992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:55:29.595427171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:55:34.265003673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:52638"
ts=2026-04-22T19:55:35.909787143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:56:04.265835525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60730"
ts=2026-04-22T19:56:05.910974485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T19:56:20.310412794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:56:31.843914448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43516"
ts=2026-04-22T19:56:34.267078561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39036"
ts=2026-04-22T19:56:35.911994055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:57:01.845057485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49198"
level=debug ts=2026-04-22T19:57:01.856673654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:57:04.268030556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:45388"
ts=2026-04-22T19:57:05.913625884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:57:35.914410428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T19:57:36.25677964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45766"
level=debug ts=2026-04-22T19:57:56.526849298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:58:04.270147617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46624"
ts=2026-04-22T19:58:05.91579681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:58:31.848427136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47124"
ts=2026-04-22T19:58:35.917390989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T19:58:36.259154523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41178"
level=debug ts=2026-04-22T19:58:55.573665184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:58:55.57522589Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:59:01.850269015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51316"
ts=2026-04-22T19:59:05.919150753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T19:59:34.274557522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:48134"
ts=2026-04-22T19:59:35.920094863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T19:59:39.614886208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:00:01.852339271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43020"
ts=2026-04-22T20:00:05.921654156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=error ts=2026-04-22T20:00:15.292700623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:00:15.294546668Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:00:15.296198371Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:00:15.296232317Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T20:00:16.222420717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:00:35.92278662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:01:01.855769527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52606"
ts=2026-04-22T20:01:05.924506878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:01:09.121352167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:01:35.925470463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:01:54.948502614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:02:04.280198188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:58390"
ts=2026-04-22T20:02:05.927200201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:02:35.928380207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:02:36.26931617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57456"
level=debug ts=2026-04-22T20:02:50.322054316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:02:50.323584691Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:03:01.861869879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60804"
ts=2026-04-22T20:03:05.929495647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:03:25.203619037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:03:35.930748421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:04:05.932152099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:04:06.272690447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36536"
level=debug ts=2026-04-22T20:04:19.269843143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:04:34.287261727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54588"
ts=2026-04-22T20:04:35.933837716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:04:36.273954892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48260"
ts=2026-04-22T20:05:04.287886258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60776"
ts=2026-04-22T20:05:05.9350789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:05:18.532954819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:05:35.936775596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:05:36.276694519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47206"
ts=2026-04-22T20:06:04.290318565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60110"
ts=2026-04-22T20:06:05.938512256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:06:16.374994747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:06:31.871369684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37816"
ts=2026-04-22T20:06:35.939468552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:07:01.87268031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54618"
ts=2026-04-22T20:07:05.940296359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:07:15.147428373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:07:15.148983586Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:07:35.941856185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:07:53.780652638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:08:05.942954341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:08:31.876847795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34956"
ts=2026-04-22T20:08:35.944023258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:08:36.283836246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36818"
level=debug ts=2026-04-22T20:08:38.629870954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:09:01.878011419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34820"
ts=2026-04-22T20:09:05.945449488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:09:06.285108745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48926"
ts=2026-04-22T20:09:31.879082881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40080"
level=debug ts=2026-04-22T20:09:35.599723811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:09:35.947023247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:10:01.88085394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49468"
ts=2026-04-22T20:10:05.948700918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=error ts=2026-04-22T20:10:15.293717994Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:10:15.295435042Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:10:15.297142152Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:10:15.297173362Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T20:10:26.403301474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:10:34.301267648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:53520"
ts=2026-04-22T20:10:35.949883163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:10:36.289552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56490"
ts=2026-04-22T20:11:05.951154016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:11:06.291124557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50852"
level=debug ts=2026-04-22T20:11:09.892256848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:11:09.894397577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:11:31.884592619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59396"
ts=2026-04-22T20:11:35.952181114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:11:44.839807698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:12:01.885581782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53632"
ts=2026-04-22T20:12:05.953706597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:12:06.294554949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33086"
level=debug ts=2026-04-22T20:12:22.323042917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:12:35.95474331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:13:01.887579297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39138"
ts=2026-04-22T20:13:05.955840492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:13:06.297206613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39902"
level=debug ts=2026-04-22T20:13:15.059361157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:13:31.88863937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42040"
ts=2026-04-22T20:13:35.956966158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:14:00.973069724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:14:01.890224334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35004"
ts=2026-04-22T20:14:04.311619833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42688"
ts=2026-04-22T20:14:05.958763827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:14:31.892199275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39948"
ts=2026-04-22T20:14:35.960335707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:14:58.345401446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:14:58.347073252Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:15:05.961714092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:15:06.304033514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48860"
ts=2026-04-22T20:15:31.89422704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51686"
ts=2026-04-22T20:15:34.320013575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54076"
ts=2026-04-22T20:15:35.963341186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:15:36.305512513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34786"
level=debug ts=2026-04-22T20:15:53.587524728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:16:01.895090559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55950"
ts=2026-04-22T20:16:04.321712296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:42002"
ts=2026-04-22T20:16:05.964723879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:16:06.306326689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34924"
ts=2026-04-22T20:16:31.896338482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56552"
ts=2026-04-22T20:16:34.322865255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:37030"
ts=2026-04-22T20:16:35.965554318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:16:36.307149146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46668"
level=debug ts=2026-04-22T20:16:48.440414162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:17:04.32380026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54580"
ts=2026-04-22T20:17:05.966589922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:17:31.898604402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42770"
ts=2026-04-22T20:17:35.967640224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:17:36.309541818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58260"
level=debug ts=2026-04-22T20:17:44.804973167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:04.325949112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:58058"
ts=2026-04-22T20:18:05.969241571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:18:06.310445046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53950"
level=debug ts=2026-04-22T20:18:30.918389192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:31.901683759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47374"
ts=2026-04-22T20:18:35.970235947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:19:05.971830767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:19:06.313502132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38018"
level=debug ts=2026-04-22T20:19:31.441354125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:19:31.443006889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:19:35.973079859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:20:05.974976966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:20:14.057305865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:20:15.293873165Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:20:15.295729428Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:20:15.297715659Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:20:15.297744239Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:20:31.907024068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45048"
ts=2026-04-22T20:20:35.976025096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:20:36.317742681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53820"
ts=2026-04-22T20:21:05.976938988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:21:12.999745273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:21:35.978038686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:21:49.419931265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:22:05.979013134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:22:06.321154377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50922"
ts=2026-04-22T20:22:31.912246701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47466"
ts=2026-04-22T20:22:35.979952383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:22:46.997038366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:23:04.341510099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:46044"
ts=2026-04-22T20:23:05.98130452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:23:35.982967514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:23:38.775934506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:23:38.777533949Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:24:04.344094615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40722"
ts=2026-04-22T20:24:05.984506858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:24:31.379854299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:35.986129811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:24:36.328153549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37724"
ts=2026-04-22T20:25:05.987325408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:25:17.203214221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:25:35.988450803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:25:58.394497977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:26:05.98999464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:26:06.332724445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53410"
ts=2026-04-22T20:26:34.350564962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39006"
level=debug ts=2026-04-22T20:26:34.663193645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:26:35.991786335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:26:36.333634077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59322"
ts=2026-04-22T20:27:01.925293962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54554"
ts=2026-04-22T20:27:05.993443369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
level=debug ts=2026-04-22T20:27:22.575816508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:27:22.577657172Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:27:34.353365703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:38092"
ts=2026-04-22T20:27:35.995028976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:27:36.336742971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52510"
ts=2026-04-22T20:28:01.928195897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37790"
ts=2026-04-22T20:28:04.354864828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:44940"
ts=2026-04-22T20:28:05.995992151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:28:19.708448013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:28:34.356823654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:60376"
ts=2026-04-22T20:28:35.997311219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0f1fecd8 10.128.0.79:7946"
ts=2026-04-22T20:28:36.339988716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33714"
level=debug ts=2026-04-22T20:28:56.268814169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:29:01.930274601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51906"
ts=2026-04-22T20:29:05.998330528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:29:35.99916936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:29:43.96512761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:30:06.001005811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=error ts=2026-04-22T20:30:15.294213169Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:30:15.296167226Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:30:15.298146186Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:30:15.298171101Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:30:36.002568654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:30:44.451313643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:31:06.00395301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:31:06.34761331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60736"
level=debug ts=2026-04-22T20:31:22.968567819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:31:22.9701659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:31:34.364028857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:39450"
ts=2026-04-22T20:31:36.004821454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:32:00.770270848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:06.005823223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:32:06.349739875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37106"
ts=2026-04-22T20:32:16.040182981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43506"
ts=2026-04-22T20:32:16.045396516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43508"
ts=2026-04-22T20:32:36.006929342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
level=debug ts=2026-04-22T20:32:36.010848843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:36.073147545Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0f1fecd8 (timeout reached)"
ts=2026-04-22T20:32:39.073158241Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0f1fecd8 has failed, no acks received"
ts=2026-04-22T20:32:44.08642613Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-0f1fecd8 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:33:06.007861038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:33:36.009568832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:33:37.925443822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:34:06.011426548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:34:14.193419672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:34:31.944352626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43564"
ts=2026-04-22T20:34:34.372381248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:51454"
ts=2026-04-22T20:34:36.01332593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
level=debug ts=2026-04-22T20:34:49.969229718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:34:49.970917886Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:35:01.946026826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33662"
ts=2026-04-22T20:35:06.014860041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
level=debug ts=2026-04-22T20:35:26.716662551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:35:34.374354631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:47946"
ts=2026-04-22T20:35:36.016303237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
ts=2026-04-22T20:36:01.948106046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33526"
ts=2026-04-22T20:36:02.961403509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47960"
ts=2026-04-22T20:36:04.37532284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:40116"
ts=2026-04-22T20:36:06.017691103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:36:23.447426066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:36:36.019490037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
ts=2026-04-22T20:37:04.377356614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:54190"
ts=2026-04-22T20:37:06.020966255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:37:08.661456706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:37:31.95165337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51094"
ts=2026-04-22T20:37:36.022367792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
ts=2026-04-22T20:38:01.952476557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52672"
ts=2026-04-22T20:38:02.966043775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52532"
ts=2026-04-22T20:38:04.379360234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:41678"
ts=2026-04-22T20:38:06.023323488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
level=debug ts=2026-04-22T20:38:09.912027543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:38:36.024958491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5vbj6-96e6be79 10.128.0.74:7946"
ts=2026-04-22T20:39:02.96909356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43594"
level=debug ts=2026-04-22T20:39:03.603433904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:39:03.605160977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:39:06.026090006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
ts=2026-04-22T20:39:34.383550458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.76:55920"
ts=2026-04-22T20:39:36.026933275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j9krs-f7aa84c2 10.128.0.76:7946"
level=debug ts=2026-04-22T20:39:53.180819988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:40:06.028256078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-70d4b141 10.128.0.79:7946"
level=error ts=2026-04-22T20:40:15.294708668Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:40:15.29717077Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:40:15.299305962Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:40:15.299344393Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"