level=info ts=2026-04-17T21:39:35.813473306Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T21:39:35.813516703Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T21:39:35.815141094Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T21:39:35.81550299Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T21:39:35.815615082Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.81 inf=eth0
level=debug ts=2026-04-17T21:39:35.816894583Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T21:39:35.817028176Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.81 inf=eth0
level=info ts=2026-04-17T21:39:35.817510086Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-5acf97a7
ts=2026-04-17T21:39:35.817599081Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T21:39:35.81761327Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.81 advertisePort=7946
level=debug ts=2026-04-17T21:39:35.817626431Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.81 advertisePort=7946
level=info ts=2026-04-17T21:39:35.817776738Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T21:39:35.827896433Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T21:39:35.827925335Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T21:39:35.827939748Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=10.165784ms
level=error ts=2026-04-17T21:39:35.827948989Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T21:39:35.827962708Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T21:39:35.830925736Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T21:39:35.830945852Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T21:39:35.833629099Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-17T21:39:35.833635747Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-17T21:39:35.83368701Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-17T21:39:35.833694447Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-17T21:39:35.833699336Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-17T21:39:35.833707414Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-17T21:39:35.834050343Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-17T21:39:35.834062936Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-17T21:39:35.834069159Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-17T21:39:35.83408957Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T21:39:35.834152759Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T21:39:35.834203529Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T21:39:35.834226423Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T21:39:35.834401596Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T21:39:35.834410653Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T21:39:35.834427146Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T21:39:36.835115079Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-17T21:39:36.944400865Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T21:39:36.944486146Z caller=loki.go:599 msg="Loki started" startup_time=1.202288261s
ts=2026-04-17T21:39:37.142237444Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T21:39:37.142276081Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:39:39.203990138Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T21:39:39.204030189Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T21:39:41.944696976Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T21:39:41.944792634Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T21:39:43.633205501Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T21:39:43.633240404Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T21:39:49.412262232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:39:57.638955356Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
ts=2026-04-17T21:39:57.639115563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-17T21:39:57.639144232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52028"
ts=2026-04-17T21:39:57.640340488Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.82:7946: dial tcp 10.129.0.82:7946: connect: connection refused"
level=info ts=2026-04-17T21:39:57.640363273Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.812395438s
ts=2026-04-17T21:40:00.463319077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41638"
ts=2026-04-17T21:40:01.91247828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44232"
level=debug ts=2026-04-17T21:40:05.218130228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:40:06.324242848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:39242"
level=debug ts=2026-04-17T21:40:28.303906729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:40:30.59519648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:40:42.472224456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:45146"
ts=2026-04-17T21:40:49.894513549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36636"
level=debug ts=2026-04-17T21:40:53.617810864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:41:00.598737851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T21:41:21.406129283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T21:41:21.40809548Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:41:22.959598368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56634"
ts=2026-04-17T21:41:30.60122109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:42:00.603273746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:42:02.881132126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:42:12.484027457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:38976"
ts=2026-04-17T21:42:22.964217543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44946"
ts=2026-04-17T21:42:30.604298483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:42:49.905691248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42342"
ts=2026-04-17T21:43:00.60543294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:43:11.946389337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:43:19.908860345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35770"
ts=2026-04-17T21:43:30.608051747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T21:43:49.9126653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39704"
ts=2026-04-17T21:44:00.610186861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:44:03.652229817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:44:19.916075365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55540"
ts=2026-04-17T21:44:30.613120459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:44:42.49687542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51400"
ts=2026-04-17T21:44:49.919807432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45160"
ts=2026-04-17T21:45:00.615850798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:45:05.527822091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T21:45:19.923131173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34236"
ts=2026-04-17T21:45:30.61812864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:45:49.926214849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46382"
level=debug ts=2026-04-17T21:45:56.507999632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T21:45:56.509973415Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:46:00.622244011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:46:22.991352577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:34736"
ts=2026-04-17T21:46:30.62542902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T21:46:42.508745375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53692"
ts=2026-04-17T21:46:49.931804158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39172"
level=debug ts=2026-04-17T21:46:52.227285976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:47:00.628051214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:47:12.512366739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43620"
ts=2026-04-17T21:47:30.629302335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:47:49.938493401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53142"
level=debug ts=2026-04-17T21:47:49.985952054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:48:00.6302984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:48:23.004579694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59764"
level=debug ts=2026-04-17T21:48:29.640944685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:48:30.632279437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:48:42.519514625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:37188"
ts=2026-04-17T21:48:49.943221928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49564"
ts=2026-04-17T21:49:00.635145952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:49:12.522929072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:33350"
level=debug ts=2026-04-17T21:49:26.462885927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:49:30.639028694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=info ts=2026-04-17T21:49:41.945455162Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-17T21:49:41.951955941Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:49:41.954284238Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:49:41.956327735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:49:41.956388146Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T21:49:41.956404609Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-17T21:49:49.950950667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41884"
ts=2026-04-17T21:50:00.642325206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T21:50:06.313629127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T21:50:06.315405224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:50:12.527961362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:35410"
ts=2026-04-17T21:50:30.644938229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:50:46.039782371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:50:49.955977403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38056"
ts=2026-04-17T21:50:53.020469696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43644"
ts=2026-04-17T21:51:00.646916971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:51:23.022090113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48860"
level=debug ts=2026-04-17T21:51:27.18522397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:51:30.649772916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:51:53.023887549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36070"
ts=2026-04-17T21:52:00.652004421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:52:03.376715594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:52:30.654720155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T21:52:42.540033001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40584"
ts=2026-04-17T21:53:00.658162257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T21:53:01.669579263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:53:30.662126048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T21:54:00.664968968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:54:01.899936144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T21:54:01.901944264Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:54:19.981494019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53574"
ts=2026-04-17T21:54:30.667812943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T21:54:50.932580116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:55:00.671308267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:55:30.67562866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T21:55:31.46816702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:55:42.557531952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51294"
ts=2026-04-17T21:55:49.990580197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52526"
ts=2026-04-17T21:56:00.678941143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:56:14.701913456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:56:30.68208153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T21:57:00.689162325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T21:57:15.568599311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:57:30.692143459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T21:57:42.570428611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40772"
ts=2026-04-17T21:58:00.696223891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T21:58:02.446367432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T21:58:02.448675232Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:58:12.574354307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:60128"
ts=2026-04-17T21:58:20.006051198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44174"
ts=2026-04-17T21:58:30.699045557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:58:42.578187986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:37622"
level=debug ts=2026-04-17T21:58:56.104903159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T21:59:00.701181228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T21:59:23.080042082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44742"
ts=2026-04-17T21:59:30.703401835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=error ts=2026-04-17T21:59:41.961110904Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:59:41.962982848Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:59:41.96481758Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T21:59:41.964865389Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T21:59:50.014914065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34904"
ts=2026-04-17T21:59:53.081825846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39570"
level=debug ts=2026-04-17T21:59:53.170935178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:00:00.706774814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:00:23.08374995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39752"
ts=2026-04-17T22:00:30.710404902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:00:42.585421817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:50494"
level=debug ts=2026-04-17T22:00:49.851276436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:00:50.022766673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37810"
ts=2026-04-17T22:00:53.085956226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60816"
ts=2026-04-17T22:01:00.712501482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T22:01:30.713734944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:01:42.294123496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:01:42.597761092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:58960"
ts=2026-04-17T22:01:53.090546342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59386"
ts=2026-04-17T22:02:00.716768126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:02:28.32024447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:02:28.322530146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:02:30.720169189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:03:00.724147673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:03:06.24682235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:03:23.099781761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:53962"
ts=2026-04-17T22:03:30.726344555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T22:03:42.606635189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51866"
ts=2026-04-17T22:03:53.101360123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44830"
ts=2026-04-17T22:04:00.728494449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:04:02.819719148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:04:23.103148713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49668"
ts=2026-04-17T22:04:30.732690901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:04:50.045980345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52720"
level=debug ts=2026-04-17T22:04:56.255507355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:05:00.735468755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T22:05:23.108946247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:34624"
ts=2026-04-17T22:05:30.738127583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:05:44.143277008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:06:00.742349723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:06:30.745807117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:06:33.861902879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:06:33.863591997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:06:50.058160619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:32968"
ts=2026-04-17T22:06:53.117088509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42222"
ts=2026-04-17T22:07:00.74984407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:07:10.976787902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:07:12.626744672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:48580"
ts=2026-04-17T22:07:30.752553585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T22:07:42.630545395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:46470"
ts=2026-04-17T22:08:00.75379363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:08:09.238811321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:08:12.634062523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:54722"
ts=2026-04-17T22:08:30.756089639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:08:50.071602781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49374"
ts=2026-04-17T22:09:00.758487674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:09:04.725431447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:09:30.761030544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=error ts=2026-04-17T22:09:41.960556615Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:09:41.962388917Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:09:41.964159972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:09:41.964188127Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:09:53.13668215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49414"
level=debug ts=2026-04-17T22:09:56.047212363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:10:00.765338663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:10:30.769236506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:10:56.61145068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:10:56.613338126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:11:00.773150243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:11:12.651523299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:34098"
ts=2026-04-17T22:11:30.775455727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:11:44.882482827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:12:00.778256882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:12:23.153231659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33604"
level=debug ts=2026-04-17T22:12:28.985459291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:12:30.78227442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:13:00.785830285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:13:12.664661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55346"
level=debug ts=2026-04-17T22:13:17.982990277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:13:30.789579108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:13:42.66814331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43196"
ts=2026-04-17T22:14:00.792511932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:14:08.972603181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:14:20.101038749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:56030"
ts=2026-04-17T22:14:23.166210031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35346"
ts=2026-04-17T22:14:30.795766118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:15:00.797867931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:15:04.480133837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:15:04.482228575Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:15:23.170962113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41262"
ts=2026-04-17T22:15:30.799610185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:15:40.563320973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:15:53.172446179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57136"
ts=2026-04-17T22:16:00.802279031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:16:16.243840433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:16:23.17378472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41184"
ts=2026-04-17T22:16:30.805979781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:16:50.115526344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46812"
ts=2026-04-17T22:17:00.80792797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:17:10.454311107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:17:20.118797701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50480"
ts=2026-04-17T22:17:30.810143601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:17:42.705885505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43930"
ts=2026-04-17T22:18:00.813958257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:18:10.477868743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:18:23.191302764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52652"
ts=2026-04-17T22:18:30.817693551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:18:42.712967365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53108"
ts=2026-04-17T22:18:53.192529361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39350"
ts=2026-04-17T22:19:00.820932269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:19:02.07298459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:19:02.075132317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:19:30.823746451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=error ts=2026-04-17T22:19:41.961840418Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:19:41.96432027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:19:41.966605487Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T22:19:41.966675307Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:19:50.130388574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51378"
ts=2026-04-17T22:19:53.19790053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43004"
ts=2026-04-17T22:20:00.827046412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
level=debug ts=2026-04-17T22:20:03.914795832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:20:30.831054344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:20:53.202783096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56174"
ts=2026-04-17T22:21:00.833837229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:21:03.625454569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:21:20.141463957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47474"
ts=2026-04-17T22:21:30.837073487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:21:47.671090135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:22:00.840822366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:22:30.842996617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
ts=2026-04-17T22:22:42.730357779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:50296"
level=debug ts=2026-04-17T22:22:44.781605318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:23:00.845974705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
ts=2026-04-17T22:23:20.149675014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42846"
ts=2026-04-17T22:23:30.849698614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:23:40.689839564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T22:23:40.692039019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T22:23:42.737676956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:38208"
ts=2026-04-17T22:23:50.153524908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42200"
ts=2026-04-17T22:24:00.85289318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"
level=debug ts=2026-04-17T22:24:17.3361532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:24:20.15768518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42678"
ts=2026-04-17T22:24:23.227468266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37294"
ts=2026-04-17T22:24:30.856367129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pdr89-7647d6fa 10.130.0.72:7946"
ts=2026-04-17T22:24:50.161578754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53574"
ts=2026-04-17T22:25:00.859344025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9803dd6a 10.129.0.83:7946"
level=debug ts=2026-04-17T22:25:05.369470995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T22:25:20.165381389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46808"
ts=2026-04-17T22:25:30.861684908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8whgw-c54d65d1 10.130.0.75:7946"