level=info ts=2026-04-21T04:01:53.162202484Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T04:01:53.162251143Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T04:01:53.163482403Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T04:01:53.163774738Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T04:01:53.163852232Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.80 inf=eth0
level=debug ts=2026-04-21T04:01:53.164566608Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T04:01:53.164632581Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.80 inf=eth0
level=info ts=2026-04-21T04:01:53.164720175Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c993d3c7
ts=2026-04-21T04:01:53.164841449Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T04:01:53.164856981Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.80 advertisePort=7946
level=debug ts=2026-04-21T04:01:53.164875761Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.80 advertisePort=7946
level=info ts=2026-04-21T04:01:53.16503513Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T04:01:53.171736885Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T04:01:53.171752086Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:01:53.171761319Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.728873ms
level=error ts=2026-04-21T04:01:53.171768776Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T04:01:53.171779455Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T04:01:53.174576135Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:53.174595572Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T04:01:53.178657328Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T04:01:53.178664648Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T04:01:53.178674368Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T04:01:53.178703775Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T04:01:53.178703644Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T04:01:53.178719719Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T04:01:53.178806005Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T04:01:53.178821306Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T04:01:53.178953708Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T04:01:53.178972717Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T04:01:53.179026193Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T04:01:53.179042962Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T04:01:53.179049892Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T04:01:53.179196442Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T04:01:53.179233137Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T04:01:53.179261193Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T04:01:54.179505819Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T04:01:54.314747913Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T04:01:54.314816903Z caller=loki.go:599 msg="Loki started" startup_time=1.215510247s
ts=2026-04-21T04:01:54.785392833Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:54.785417376Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:01:58.47773279Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:58.477765602Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T04:01:59.315880316Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T04:01:59.315940222Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T04:02:04.898304713Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:02:04.898328923Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:02:13.894085582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40504"
ts=2026-04-21T04:02:15.212623922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58964"
ts=2026-04-21T04:02:15.360828102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46762"
level=debug ts=2026-04-21T04:02:16.884492568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:02:18.376454617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.80:7946"
ts=2026-04-21T04:02:18.376559985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:45418"
level=info ts=2026-04-21T04:02:18.377026936Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=25.205243345s
level=debug ts=2026-04-21T04:02:37.743939283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:02:45.531242649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:02:57.080002898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:03:06.951632017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35748"
ts=2026-04-21T04:03:15.53460931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:03:19.672872565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:03:44.037659702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50528"
ts=2026-04-21T04:03:45.538026924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:03:53.621736115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:03:53.623348817Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:04:15.541365508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:04:24.857975492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59684"
level=debug ts=2026-04-21T04:04:32.536296933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:04:45.544826846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:05:15.547859916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:05:40.874142558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:05:45.551116345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:06:15.554017234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:06:24.868780271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50622"
ts=2026-04-21T04:06:45.556931527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:06:48.977144634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:06:54.872182978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:44446"
ts=2026-04-21T04:07:06.973605141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53290"
ts=2026-04-21T04:07:15.55917251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:07:34.087671526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:07:36.974786704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48972"
ts=2026-04-21T04:07:45.561160118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:08:06.976402333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50672"
ts=2026-04-21T04:08:15.563193503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:08:35.731148159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:08:35.732765428Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:08:36.977437915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59544"
ts=2026-04-21T04:08:45.565162796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:09:12.1339558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:09:15.568306992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:09:36.981428728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40398"
ts=2026-04-21T04:09:45.570728123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:10:06.982668749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55634"
level=debug ts=2026-04-21T04:10:13.332365587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:10:15.573395957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:10:36.984170622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37300"
ts=2026-04-21T04:10:45.575940607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:10:54.892921651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:33162"
level=debug ts=2026-04-21T04:11:05.40410199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:11:15.577380666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:11:45.579322852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T04:11:59.316767593Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T04:11:59.320475216Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:11:59.322455967Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:11:59.324200427Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:11:59.324247046Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T04:11:59.324260871Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-21T04:12:04.666059834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:12:15.581127729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:12:36.994563893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42634"
ts=2026-04-21T04:12:44.078374918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46906"
ts=2026-04-21T04:12:45.582434911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:12:45.626357065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:12:45.627947464Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:12:54.901507571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36978"
ts=2026-04-21T04:13:15.584867929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:13:24.904382196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52474"
level=debug ts=2026-04-21T04:13:35.288009816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:13:45.587480774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:14:10.301421954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:14:14.083361964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47392"
ts=2026-04-21T04:14:15.590901137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:14:44.086012063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56420"
ts=2026-04-21T04:14:45.592808676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:14:50.478800055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:15:15.59479388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:15:45.597418269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:15:52.038559828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:15:54.915952678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58692"
ts=2026-04-21T04:16:07.013373002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54346"
ts=2026-04-21T04:16:15.599342257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:16:41.741636038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:16:41.743257063Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:16:44.098523339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47506"
ts=2026-04-21T04:16:45.602051091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:16:54.91993095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:38310"
ts=2026-04-21T04:17:14.101660408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49226"
ts=2026-04-21T04:17:15.60502899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:17:23.267713786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:17:24.922755617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52310"
ts=2026-04-21T04:17:45.607980848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:18:15.611160432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:18:24.699842534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:18:24.928597709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40724"
ts=2026-04-21T04:18:37.026743801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44410"
ts=2026-04-21T04:18:45.613867008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:19:07.027977648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55228"
ts=2026-04-21T04:19:15.6165838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:19:21.99549852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:19:24.93526584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42766"
ts=2026-04-21T04:19:37.028734723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36456"
ts=2026-04-21T04:19:45.619365958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:20:01.250199735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:20:07.029977536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49962"
ts=2026-04-21T04:20:14.112490075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52806"
ts=2026-04-21T04:20:15.621789436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:20:24.93938277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48438"
ts=2026-04-21T04:20:37.030999668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48350"
ts=2026-04-21T04:20:45.623558503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:21:02.192112846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:21:02.193717004Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:21:14.116726123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34192"
ts=2026-04-21T04:21:15.624585701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:21:37.035800225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33976"
ts=2026-04-21T04:21:44.119484169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39838"
ts=2026-04-21T04:21:45.626197761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:21:57.311681874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:21:59.326762924Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:21:59.328638321Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:21:59.330324027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:21:59.330359195Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:22:07.037507124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42726"
ts=2026-04-21T04:22:14.122708505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56094"
ts=2026-04-21T04:22:15.628406481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:22:37.039501157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39980"
level=debug ts=2026-04-21T04:22:41.396107397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:22:44.125880074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42694"
ts=2026-04-21T04:22:45.631064154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:23:07.041033159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38170"
ts=2026-04-21T04:23:14.129343006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44858"
ts=2026-04-21T04:23:15.633056487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:23:24.952347379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:34692"
level=debug ts=2026-04-21T04:23:35.39187316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:23:45.636498304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:24:15.006334516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:24:15.638178013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:24:45.640097849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:25:00.49734555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:25:00.499002577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:25:15.64249019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:25:44.141509191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57596"
ts=2026-04-21T04:25:45.645350356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:25:50.734004565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:26:07.058193035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59974"
ts=2026-04-21T04:26:14.144769657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43756"
ts=2026-04-21T04:26:15.648089575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:26:24.966811746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39174"
ts=2026-04-21T04:26:37.059909498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38424"
ts=2026-04-21T04:26:44.148429008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48422"
ts=2026-04-21T04:26:45.651369102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:26:50.346881738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:27:07.061880312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46544"
ts=2026-04-21T04:27:15.654188981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:27:26.899818512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:27:45.657320546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:28:14.158065418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51144"
ts=2026-04-21T04:28:15.658954955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:28:25.488990815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:28:44.16056466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55264"
ts=2026-04-21T04:28:45.661503726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:28:54.979881939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40148"
ts=2026-04-21T04:29:07.072376608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56404"
level=debug ts=2026-04-21T04:29:08.152223444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:29:08.153777968Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:29:14.163028278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52342"
ts=2026-04-21T04:29:15.66364951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:29:24.982838827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54892"
ts=2026-04-21T04:29:45.66489183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:29:54.985846884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56996"
level=debug ts=2026-04-21T04:30:06.357659409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:30:07.076791135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34274"
ts=2026-04-21T04:30:15.667224583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:30:45.669797703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:30:54.990232903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40830"
level=debug ts=2026-04-21T04:31:01.574182378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:31:15.672571873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:31:37.084098904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44868"
ts=2026-04-21T04:31:45.675944177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:31:46.428117787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:31:59.330028117Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:31:59.331887779Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:31:59.333740358Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:31:59.333765383Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:32:15.6777214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:32:24.998700929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47420"
level=debug ts=2026-04-21T04:32:36.554196744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:32:37.08831079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46772"
ts=2026-04-21T04:32:45.679539052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:33:07.090009947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47456"
ts=2026-04-21T04:33:15.682853055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:33:26.641865764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:33:26.643628462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:33:44.182266472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44424"
ts=2026-04-21T04:33:45.685693583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:34:14.184734427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58402"
ts=2026-04-21T04:34:15.688175541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:34:27.598131798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:34:37.098411536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55984"
ts=2026-04-21T04:34:44.188290749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50812"
ts=2026-04-21T04:34:45.690064776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:35:06.847085812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:35:15.692633331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:35:44.193133882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40910"
ts=2026-04-21T04:35:45.694847877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:35:55.012048964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48752"
level=debug ts=2026-04-21T04:36:04.122470774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:36:15.697616228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:36:37.110479064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33712"
ts=2026-04-21T04:36:45.700818173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:36:47.700251928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:37:07.111994302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54630"
ts=2026-04-21T04:37:15.704193368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:37:22.50916354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:37:22.511008255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:37:25.018385759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56480"
ts=2026-04-21T04:37:37.113921815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40688"
ts=2026-04-21T04:37:44.203462944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51664"
ts=2026-04-21T04:37:45.706700657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:38:15.709368908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:38:21.935857753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:38:37.118344143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60974"
ts=2026-04-21T04:38:45.712898485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:38:55.024949427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53502"
ts=2026-04-21T04:39:15.715735391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:39:22.595432261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:39:37.123082727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39936"
ts=2026-04-21T04:39:45.718393538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:40:05.775803945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:40:15.720227927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:40:40.063286318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:40:45.723034425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:41:15.726126348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:41:25.036280537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54118"
level=debug ts=2026-04-21T04:41:26.991265063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:41:26.993027138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:41:37.133858145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38048"
ts=2026-04-21T04:41:45.729185753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=error ts=2026-04-21T04:41:59.330752678Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:41:59.332794047Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:41:59.334595916Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:41:59.33462412Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:42:06.899468401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:42:07.135144372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33550"
ts=2026-04-21T04:42:15.732268505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:42:25.04214244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41852"
ts=2026-04-21T04:42:37.136706912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54646"
ts=2026-04-21T04:42:44.228121655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54118"
ts=2026-04-21T04:42:45.734835341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:43:05.409586452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:43:14.231583413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59480"
ts=2026-04-21T04:43:15.737156285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:43:45.572952109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:43:45.740436796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:43:55.048045266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:55938"
ts=2026-04-21T04:44:15.742990162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:44:25.051083451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36114"
level=debug ts=2026-04-21T04:44:29.086295542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:44:37.147197449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34082"
ts=2026-04-21T04:44:45.744689156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:45:07.148178832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58334"
ts=2026-04-21T04:45:15.745770277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:45:15.937998164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:45:15.939905396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:45:44.24296361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40194"
ts=2026-04-21T04:45:45.747840864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:46:03.119357698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:46:15.751313091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:46:25.057497451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42220"
ts=2026-04-21T04:46:37.15668712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56988"
ts=2026-04-21T04:46:44.247479024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38770"
level=debug ts=2026-04-21T04:46:45.412981503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:46:45.753972109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:47:15.757187683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:47:25.061739241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:47:45.760391909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:48:10.266286943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:48:15.762169835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:48:44.259661723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47282"
ts=2026-04-21T04:48:45.764578689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:48:48.972751129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:48:48.97464762Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:48:55.065030115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43246"
ts=2026-04-21T04:49:07.171303677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48824"
ts=2026-04-21T04:49:14.2621838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51436"
ts=2026-04-21T04:49:15.767818541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:49:40.534161751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:49:45.771167777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:49:55.071117105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36360"
ts=2026-04-21T04:50:07.176260067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41606"
ts=2026-04-21T04:50:15.774069816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:50:35.063242879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:50:37.177773149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53556"
ts=2026-04-21T04:50:44.268686109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43660"
ts=2026-04-21T04:50:45.776496064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:51:07.179378576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48482"
ts=2026-04-21T04:51:15.779031398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:51:35.028732116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:51:45.780755554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:51:55.079923824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49762"
level=error ts=2026-04-21T04:51:59.329093339Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:51:59.331233555Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:51:59.333004461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T04:51:59.333026984Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:52:15.782856177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:52:34.795727136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:52:45.785567098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:52:55.08487643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40824"
ts=2026-04-21T04:53:07.189948553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50720"
ts=2026-04-21T04:53:14.280279463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57248"
ts=2026-04-21T04:53:15.788656213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:53:31.268232754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:53:31.270050568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:53:45.791503923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:54:07.19436582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51172"
level=debug ts=2026-04-21T04:54:11.320737321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:54:15.794802688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:54:25.094425554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58322"
ts=2026-04-21T04:54:37.195298094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37574"
ts=2026-04-21T04:54:44.287570425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51582"
ts=2026-04-21T04:54:45.798235944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:54:55.097813101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56764"
level=debug ts=2026-04-21T04:55:12.629136026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:55:15.801357698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T04:55:45.80480447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:56:02.332132078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:56:14.294913903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50138"
ts=2026-04-21T04:56:15.806530391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:56:45.80830276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T04:56:54.205342611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:56:55.110191653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56678"
ts=2026-04-21T04:57:15.809944497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T04:57:45.811727322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:57:51.362526302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:57:51.364276594Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:58:15.814928448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:58:37.006101996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:58:37.217337284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44838"
ts=2026-04-21T04:58:45.817931638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T04:59:15.821268482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:59:34.511996011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:59:45.824463615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:00:07.224423674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42276"
ts=2026-04-21T05:00:15.826583201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:00:25.123727945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36104"
level=debug ts=2026-04-21T05:00:26.784236958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:00:45.82845207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:00:55.127116913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35360"
level=debug ts=2026-04-21T05:01:10.214588298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:01:15.831788837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:01:25.130049387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52596"
ts=2026-04-21T05:01:45.834946228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:01:55.132828558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53294"
level=debug ts=2026-04-21T05:01:58.945128927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:01:58.946951196Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:01:59.327971679Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:01:59.329713383Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:01:59.33132149Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:01:59.331347205Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:02:07.235801596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58110"
ts=2026-04-21T05:02:14.332401987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41538"
ts=2026-04-21T05:02:15.838089383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:02:25.136012661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45664"
level=debug ts=2026-04-21T05:02:35.403664489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:02:45.841466704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:03:13.390426934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:03:14.336619866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46914"
ts=2026-04-21T05:03:15.844195784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:03:25.141796533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41624"
ts=2026-04-21T05:03:37.243130117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54574"
ts=2026-04-21T05:03:45.84768211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:03:56.22963038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:04:07.244988814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43134"
ts=2026-04-21T05:04:14.341034322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35908"
ts=2026-04-21T05:04:15.851155521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:04:36.370510562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:04:37.246131941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45694"
ts=2026-04-21T05:04:45.854155694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:04:55.149504438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37602"
ts=2026-04-21T05:05:15.856185429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:05:38.57293809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:05:38.574644851Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:05:45.857874213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:05:55.155270062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43740"
ts=2026-04-21T05:06:14.351276946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45472"
ts=2026-04-21T05:06:15.859278045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:06:23.048307243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:06:25.158714928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47220"
ts=2026-04-21T05:06:44.354499952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58776"
ts=2026-04-21T05:06:45.86061946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:06:55.161727861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50726"
level=debug ts=2026-04-21T05:07:11.597694962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:07:14.359270506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34486"
ts=2026-04-21T05:07:15.862658167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:07:37.263930823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40274"
ts=2026-04-21T05:07:44.362161646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40158"
ts=2026-04-21T05:07:45.865865287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:07:48.481263824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:07:55.165405096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51526"
ts=2026-04-21T05:08:14.365520759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40844"
ts=2026-04-21T05:08:15.868257836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:08:28.662356437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:08:45.870527359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:09:10.242297793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:09:10.243970932Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:09:15.872477985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:09:25.17342161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59960"
ts=2026-04-21T05:09:37.274030052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43590"
ts=2026-04-21T05:09:45.874838944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:09:55.176502023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39672"
level=debug ts=2026-04-21T05:10:06.495293325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:10:07.275474363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59330"
ts=2026-04-21T05:10:14.37705455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40676"
ts=2026-04-21T05:10:15.877694406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:10:37.276874178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37088"
ts=2026-04-21T05:10:44.380450537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59784"
ts=2026-04-21T05:10:45.880035339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:11:04.227662319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:11:07.27782232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48446"
ts=2026-04-21T05:11:15.881795928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:11:44.3864791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33736"
ts=2026-04-21T05:11:45.884100949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:11:51.763619466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:11:59.327492201Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:11:59.329107899Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:11:59.330712561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:11:59.330740611Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:12:14.388917548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42902"
ts=2026-04-21T05:12:15.887168273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:12:25.188371596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48376"
ts=2026-04-21T05:12:37.285773286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41022"
level=debug ts=2026-04-21T05:12:38.756703781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:12:45.890347654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:13:15.893615747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:13:34.414484974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:13:34.416185909Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:13:45.896116556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:14:14.397470915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50644"
ts=2026-04-21T05:14:15.898039393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:14:31.777045921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:14:45.900224635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:15:07.299351851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58996"
ts=2026-04-21T05:15:15.90172274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:15:17.837500286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:15:25.200350949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48110"
ts=2026-04-21T05:15:45.903968767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:16:03.362835395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:16:15.90731752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:16:25.205345313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54008"
ts=2026-04-21T05:16:45.909309886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:16:58.442824676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:17:07.30959159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46084"
ts=2026-04-21T05:17:15.912043955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:17:44.965105464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:17:44.966646304Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:17:45.914751396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:17:55.213959796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36400"
ts=2026-04-21T05:18:15.917482043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:18:25.217442096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50028"
level=debug ts=2026-04-21T05:18:41.109319627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:18:45.920685195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:19:15.923803267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:19:18.2823342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:19:25.223514942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42324"
ts=2026-04-21T05:19:45.926606269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:20:14.479679669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:20:15.929817448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:20:25.227561088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49032"
ts=2026-04-21T05:20:37.328794211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48916"
ts=2026-04-21T05:20:45.931705918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:20:51.533502591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:20:55.230645623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59156"
ts=2026-04-21T05:21:07.330027709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34618"
ts=2026-04-21T05:21:15.93313027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:21:37.331454826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56784"
ts=2026-04-21T05:21:45.934783445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:21:52.544178601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:21:52.546453859Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:21:55.236444227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56962"
level=error ts=2026-04-21T05:21:59.326974049Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:21:59.328768673Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:21:59.330467474Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:21:59.330502133Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:22:07.333010715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51926"
ts=2026-04-21T05:22:14.434414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44260"
ts=2026-04-21T05:22:15.937004226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:22:25.238996482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:33214"
level=debug ts=2026-04-21T05:22:37.193881072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:22:44.437125606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:53530"
ts=2026-04-21T05:22:45.939752367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:23:15.942385267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:23:28.308049899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:23:37.341352083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34024"
ts=2026-04-21T05:23:45.945626759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:24:07.343178091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47906"
ts=2026-04-21T05:24:15.947312939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:24:25.123936779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:24:25.246238217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49160"
ts=2026-04-21T05:24:45.949312565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:25:07.347353108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57408"
ts=2026-04-21T05:25:15.952737036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:25:24.167809049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:25:37.348359066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56564"
ts=2026-04-21T05:25:45.955423624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:25:55.253628342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:40902"
level=debug ts=2026-04-21T05:26:06.119293802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:26:06.120915125Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:26:15.956988839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
ts=2026-04-21T05:26:44.456415472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46614"
ts=2026-04-21T05:26:45.959659283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:26:55.258870449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41082"
level=debug ts=2026-04-21T05:27:02.490347006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:27:07.35585294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41014"
ts=2026-04-21T05:27:15.96282042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:27:25.261595051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52112"
ts=2026-04-21T05:27:37.357257898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50738"
ts=2026-04-21T05:27:45.966161953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:27:46.930960651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:28:07.358109294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56406"
ts=2026-04-21T05:28:15.969300451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:28:29.831609696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:28:37.359665301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35110"
ts=2026-04-21T05:28:44.465882551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57986"
ts=2026-04-21T05:28:45.972457327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:29:10.209539682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:29:14.468692912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41442"
ts=2026-04-21T05:29:15.975488024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:29:44.471774117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38214"
ts=2026-04-21T05:29:45.978439788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:29:47.422680681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:29:47.424290353Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:30:07.366997738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51966"
ts=2026-04-21T05:30:15.981848686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:30:42.062556772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:30:45.984887579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:31:15.988325667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:31:18.810793452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:31:44.480105568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46516"
ts=2026-04-21T05:31:45.990793442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=error ts=2026-04-21T05:31:59.327943282Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:31:59.330078462Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:31:59.332023943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T05:31:59.332059402Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T05:32:03.3279931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:32:07.377136889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58182"
ts=2026-04-21T05:32:15.992810495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:32:45.994968872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-80814b76 10.128.0.79:7946"
level=debug ts=2026-04-21T05:32:57.250895316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:33:07.381711821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58908"
ts=2026-04-21T05:33:15.997088074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:33:25.285414966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46692"
level=debug ts=2026-04-21T05:33:40.367897352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:33:40.369428249Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T05:33:46.000588056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
ts=2026-04-21T05:34:14.491929247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40866"
ts=2026-04-21T05:34:16.003673144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:34:25.291304315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:57408"
level=debug ts=2026-04-21T05:34:28.868880365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:34:37.389298551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33466"
ts=2026-04-21T05:34:46.00660775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
ts=2026-04-21T05:35:16.009968424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:35:16.431234683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"