level=info ts=2026-02-16T18:01:57.086729215Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T18:01:57.08680645Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T18:01:57.088347025Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T18:01:57.088651648Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:01:57.08872898Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.61 inf=eth0
level=debug ts=2026-02-16T18:01:57.089575885Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T18:01:57.089656386Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.61 inf=eth0
level=info ts=2026-02-16T18:01:57.090734852Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c9ed856e
ts=2026-02-16T18:01:57.090821813Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T18:01:57.09083857Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.61 advertisePort=7946
level=debug ts=2026-02-16T18:01:57.090863175Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.61 advertisePort=7946
level=info ts=2026-02-16T18:01:57.091031091Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T18:01:57.098394017Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:01:57.098416072Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T18:01:57.098430734Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.402448ms
level=error ts=2026-02-16T18:01:57.098441414Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T18:01:57.098452601Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T18:01:57.10304772Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:57.103067971Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T18:01:57.104024843Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T18:01:57.104040612Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-02-16T18:01:57.104086482Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T18:01:57.104077726Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-02-16T18:01:57.104096543Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T18:01:57.104132912Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T18:01:57.104225396Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-02-16T18:01:57.104241849Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T18:01:57.104397976Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-02-16T18:01:57.104406576Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T18:01:57.104455822Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T18:01:57.104455772Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T18:01:57.104477827Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T18:01:57.104671931Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T18:01:57.104687946Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T18:01:57.104719067Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T18:01:58.105427659Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T18:01:58.302025155Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T18:01:58.302084688Z caller=loki.go:599 msg="Loki started" startup_time=1.283054816s
ts=2026-02-16T18:01:58.836580127Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:01:58.836606342Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:02.632163626Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:02.632189258Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T18:02:03.303121613Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T18:02:03.303182139Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T18:02:07.166102308Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:07.166134086Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T18:02:15.708099658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:17.084973197Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T18:02:17.084998201Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:34.075429744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:45936"
ts=2026-02-16T18:02:35.843890692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40254"
level=debug ts=2026-02-16T18:02:39.394240869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:02:43.224489797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53580"
ts=2026-02-16T18:02:43.438366683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49368"
ts=2026-02-16T18:02:43.721644304Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.59:7946: dial tcp 10.128.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:43.722707053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.58:7946"
ts=2026-02-16T18:02:43.725040197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-02-16T18:02:43.727197897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.55:7946"
ts=2026-02-16T18:02:43.729450569Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T18:02:43.730104093Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.62:7946: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-02-16T18:02:43.73019997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
ts=2026-02-16T18:02:43.730228959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52612"
level=info ts=2026-02-16T18:02:43.730598512Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=46.632139419s
ts=2026-02-16T18:02:54.659201041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42788"
level=debug ts=2026-02-16T18:02:58.481878775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:03:00.865452237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:03:18.706920335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:03:23.156706244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50612"
ts=2026-02-16T18:03:30.868854828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:03:50.08428269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:03:50.086103354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:04:00.872323352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:04:23.161953317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57376"
level=debug ts=2026-02-16T18:04:24.244191795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:04:24.664601874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41276"
ts=2026-02-16T18:04:30.875845079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:04:54.66739166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33956"
ts=2026-02-16T18:05:00.878654368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:05:13.232867304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:38840"
ts=2026-02-16T18:05:23.166317613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55694"
ts=2026-02-16T18:05:30.882088104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:05:35.644119192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:05:43.235725535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51910"
ts=2026-02-16T18:06:00.885144859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:06:13.238836376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50028"
ts=2026-02-16T18:06:30.887781479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:06:53.17290156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54842"
level=debug ts=2026-02-16T18:06:54.136717572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:06:54.674898629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41330"
ts=2026-02-16T18:07:00.89072567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:07:13.242512159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41844"
ts=2026-02-16T18:07:23.176198134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35794"
ts=2026-02-16T18:07:30.894247534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:07:53.825217047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T18:08:00.897454048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:08:13.24767392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:38950"
ts=2026-02-16T18:08:30.900349997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:08:43.250837048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50020"
level=debug ts=2026-02-16T18:08:47.687915956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T18:08:47.689451952Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:09:00.903279302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:09:13.254477135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43370"
ts=2026-02-16T18:09:30.906069863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:09:43.962775257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:09:54.685308495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35948"
ts=2026-02-16T18:10:00.909321622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:10:23.186395014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47992"
level=debug ts=2026-02-16T18:10:23.751082559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:10:24.688871127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47396"
ts=2026-02-16T18:10:30.91266284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:11:00.915863403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:11:08.499812043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:11:30.919185764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:11:54.69685741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57234"
ts=2026-02-16T18:12:00.922223803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=info ts=2026-02-16T18:12:03.303874325Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T18:12:03.309501081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:12:03.311625352Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:12:03.313295607Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:12:03.313333166Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T18:12:03.313346508Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-02-16T18:12:09.849596581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:12:13.26534155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40188"
ts=2026-02-16T18:12:30.925111146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:12:53.194410045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33866"
ts=2026-02-16T18:12:54.702191796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40784"
level=debug ts=2026-02-16T18:12:58.379693141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:12:58.381789218Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:13:00.928663045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:13:23.198012995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46340"
ts=2026-02-16T18:13:24.705474465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59182"
ts=2026-02-16T18:13:30.932232122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:13:41.154107208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:13:53.201471533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51954"
ts=2026-02-16T18:13:54.709099018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42752"
ts=2026-02-16T18:14:00.93591274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:14:24.713090763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58592"
ts=2026-02-16T18:14:30.939187689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:14:33.811306794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:15:00.942471572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:15:13.275261405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55646"
ts=2026-02-16T18:15:23.207966433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56636"
ts=2026-02-16T18:15:24.718619836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55072"
ts=2026-02-16T18:15:30.946039586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:15:35.289118916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:15:53.211385646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52240"
ts=2026-02-16T18:16:00.94915061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:16:25.369374846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:16:30.952334073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:16:54.725491379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33210"
ts=2026-02-16T18:17:00.955153966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:17:25.863974466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:17:25.865931642Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:17:30.95893338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:17:43.284626694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57322"
ts=2026-02-16T18:18:00.961919069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:18:13.287459355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52926"
level=debug ts=2026-02-16T18:18:18.437135091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:18:30.965319767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:19:00.96828967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:19:12.790396048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:19:24.733572189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55422"
ts=2026-02-16T18:19:30.971650854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:19:43.293043962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48224"
level=debug ts=2026-02-16T18:19:52.635560624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:20:00.974776036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:20:24.738680086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50614"
ts=2026-02-16T18:20:30.977836171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:20:39.144074809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:20:43.297034471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:37068"
ts=2026-02-16T18:21:00.980868026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:21:13.300346227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39784"
level=debug ts=2026-02-16T18:21:20.4827671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:21:20.48446563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:21:23.232703846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60512"
ts=2026-02-16T18:21:30.984379148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:21:53.235531113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58264"
ts=2026-02-16T18:22:00.990670387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=error ts=2026-02-16T18:22:03.317877252Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:22:03.320092404Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:22:03.322024052Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:22:03.322052414Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T18:22:12.689397761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:22:30.995717839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:22:43.305879728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46684"
level=debug ts=2026-02-16T18:22:49.64563813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:22:53.239957763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55044"
ts=2026-02-16T18:23:00.998952077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:23:23.242842918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46620"
ts=2026-02-16T18:23:31.002400327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:23:36.273925934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:24:01.005618429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:24:24.750321742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57338"
ts=2026-02-16T18:24:31.009214819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:24:35.425164206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:24:43.313563875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57786"
ts=2026-02-16T18:24:54.753130009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54370"
ts=2026-02-16T18:25:01.012463614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:25:20.859281101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:25:20.860940238Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:25:23.250845982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33788"
ts=2026-02-16T18:25:31.015410692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:25:43.319101487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50150"
ts=2026-02-16T18:26:01.018961095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:26:15.92238036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:26:23.255315029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45892"
ts=2026-02-16T18:26:31.021587875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:27:01.025224585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:27:13.607964525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:27:24.762211498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42428"
ts=2026-02-16T18:27:31.028354339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:27:54.765738327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55532"
ts=2026-02-16T18:28:01.031894179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:28:07.981910119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:28:13.327839518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55982"
ts=2026-02-16T18:28:24.769154423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53070"
ts=2026-02-16T18:28:31.034655669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:28:53.264658515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42690"
level=debug ts=2026-02-16T18:29:00.350700847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:29:01.038258024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:29:13.3322947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56172"
ts=2026-02-16T18:29:24.774236767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43960"
ts=2026-02-16T18:29:31.041928794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:29:51.775188658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:29:51.776818221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:30:01.044812441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:30:31.048572174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:30:43.338361668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56362"
level=debug ts=2026-02-16T18:30:46.503668518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:31:01.051304136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:31:13.341738575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50648"
ts=2026-02-16T18:31:24.781755752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57312"
ts=2026-02-16T18:31:31.054351374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:31:40.317267943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:32:01.057258216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=error ts=2026-02-16T18:32:03.319258084Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:32:03.322038666Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:32:03.324374658Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:32:03.324412982Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:32:31.060503367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:32:33.796221102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:33:01.064098299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:33:14.265364617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:33:24.788369696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33868"
ts=2026-02-16T18:33:31.067255524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:34:01.07073235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:34:04.067785532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:34:04.069558224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:34:24.793620997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34212"
ts=2026-02-16T18:34:31.073953484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:34:45.400396729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:34:54.797189412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51134"
ts=2026-02-16T18:35:01.077648717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:35:23.284921279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33650"
ts=2026-02-16T18:35:31.081052604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:35:35.858294757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:35:43.355464195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53346"
ts=2026-02-16T18:36:01.084362571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:36:23.288769764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44704"
ts=2026-02-16T18:36:24.803926631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55792"
level=debug ts=2026-02-16T18:36:28.406245954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:36:31.087303583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:36:53.29190061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51630"
ts=2026-02-16T18:37:01.090682796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:37:21.91500727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:37:31.093721263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:37:53.296325623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60782"
ts=2026-02-16T18:38:01.097031019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:38:13.364591784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:60848"
level=debug ts=2026-02-16T18:38:22.334001122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:38:22.336496263Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:38:31.100008053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:39:01.103087677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:39:22.42129109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:39:31.105946373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:39:54.816371121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40978"
ts=2026-02-16T18:40:01.109477994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:40:15.762382505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:40:31.112989742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:41:01.116268263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:41:02.204535828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:41:13.373801129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43464"
ts=2026-02-16T18:41:31.119836044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:41:53.108647762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:41:54.822562425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46430"
ts=2026-02-16T18:42:01.122750571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=error ts=2026-02-16T18:42:03.3173362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:42:03.319321153Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:42:03.321042508Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:42:03.321071448Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:42:24.826053423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55238"
ts=2026-02-16T18:42:31.125909421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:42:43.380754877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40350"
level=debug ts=2026-02-16T18:42:44.249763291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:42:44.251337819Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:42:54.82958204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56410"
ts=2026-02-16T18:43:01.128851032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:43:13.383424869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52476"
ts=2026-02-16T18:43:23.314419922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37180"
level=debug ts=2026-02-16T18:43:24.530745611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:43:31.132071721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:43:53.318129321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46094"
ts=2026-02-16T18:44:01.134666563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:44:23.320772053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43348"
level=debug ts=2026-02-16T18:44:26.411989682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:44:31.137819883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:44:43.388787657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55350"
ts=2026-02-16T18:44:53.324232718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59506"
ts=2026-02-16T18:45:01.140810305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:45:21.525137951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:45:24.838261053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56892"
ts=2026-02-16T18:45:31.143865935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:45:53.328656268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:49324"
ts=2026-02-16T18:46:01.147094121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:46:13.395421098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51124"
level=debug ts=2026-02-16T18:46:21.82828812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:46:31.150385641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:46:43.39876684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56250"
ts=2026-02-16T18:46:54.844762505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34178"
ts=2026-02-16T18:47:01.153765613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:47:13.402103679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53370"
level=debug ts=2026-02-16T18:47:18.222769106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:47:18.225204816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:47:24.847968783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38074"
ts=2026-02-16T18:47:31.157070353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:48:01.160483593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:48:03.946299365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:48:31.163236711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:48:43.40744026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40684"
ts=2026-02-16T18:48:53.341375632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33798"
level=debug ts=2026-02-16T18:48:59.584243425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:49:01.166722038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:49:23.344664352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40746"
ts=2026-02-16T18:49:24.856715067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44330"
ts=2026-02-16T18:49:31.170176827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:49:47.434139189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:50:01.172799451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:50:23.349550784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56302"
level=debug ts=2026-02-16T18:50:24.638511912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:50:31.17573391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:50:43.415047034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:60960"
ts=2026-02-16T18:51:01.179632216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:51:06.254800779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:51:06.256474857Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:51:23.353908932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55336"
ts=2026-02-16T18:51:31.182898906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:52:00.493731139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:52:01.185760254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=error ts=2026-02-16T18:52:03.316070854Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:52:03.31911303Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:52:03.321100831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T18:52:03.321128538Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:52:31.188459692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:52:59.911088311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:53:01.191992842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T18:53:13.424185208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57580"
ts=2026-02-16T18:53:23.361459657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39524"
ts=2026-02-16T18:53:24.869507596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54720"
ts=2026-02-16T18:53:31.195583562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:53:36.142722426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:54:01.199073903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:54:13.58218754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:54:31.205692891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T18:54:54.875220678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42628"
ts=2026-02-16T18:55:01.209287051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T18:55:05.471552436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:55:05.473256116Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:55:31.212991317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:55:53.369446883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33978"
ts=2026-02-16T18:56:01.216391673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:56:01.627251852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:56:13.434927216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50234"
ts=2026-02-16T18:56:23.372592765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34336"
ts=2026-02-16T18:56:31.219895561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:56:37.085907306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:56:43.438429131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40058"
ts=2026-02-16T18:56:53.375657215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55926"
ts=2026-02-16T18:57:01.223030557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:57:12.053711265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:57:13.441994219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34728"
ts=2026-02-16T18:57:23.379309246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:49268"
ts=2026-02-16T18:57:24.883425643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59372"
ts=2026-02-16T18:57:31.22589894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T18:57:48.82721531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:57:54.886417544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55910"
ts=2026-02-16T18:58:01.228854695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:58:23.384284067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41412"
ts=2026-02-16T18:58:31.232086433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T18:58:49.94387548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T18:58:49.945614151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T18:58:53.387248261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41706"
ts=2026-02-16T18:58:54.890676963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42676"
ts=2026-02-16T18:59:01.235306307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:59:13.448644763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54212"
level=debug ts=2026-02-16T18:59:26.680779696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T18:59:31.238747257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T18:59:54.895541522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50110"
ts=2026-02-16T19:00:01.241414181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:00:23.282255637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:00:24.898644503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46810"
ts=2026-02-16T19:00:31.24492902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T19:01:01.24813529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:01:15.192917807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:01:24.903015785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57978"
ts=2026-02-16T19:01:31.251357812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T19:01:43.457590165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52534"
ts=2026-02-16T19:02:01.254591999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=error ts=2026-02-16T19:02:03.316388226Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T19:02:03.318632058Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T19:02:03.320632318Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T19:02:03.320663553Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T19:02:05.507726394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:02:24.908069949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41916"
ts=2026-02-16T19:02:31.258051281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:02:40.224605713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:02:40.226279771Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T19:02:43.462561228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55136"
ts=2026-02-16T19:03:01.260723511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:03:28.475628996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:03:31.26394944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T19:03:53.401145441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58380"
ts=2026-02-16T19:04:01.267178131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
level=debug ts=2026-02-16T19:04:27.996940893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:04:31.270009805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:04:53.405848533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36482"
ts=2026-02-16T19:05:01.273060164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T19:05:23.409102064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46124"
level=debug ts=2026-02-16T19:05:25.584156887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:05:31.275928623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-xqlwd-cd02b61d 10.128.0.57:7946"
ts=2026-02-16T19:05:43.473502722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34790"
ts=2026-02-16T19:05:53.412682978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51926"
ts=2026-02-16T19:06:01.278776746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:06:13.476576892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41530"
ts=2026-02-16T19:06:23.419683973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38858"
level=debug ts=2026-02-16T19:06:27.796580558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:06:31.281768387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T19:06:54.922078391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41190"
ts=2026-02-16T19:07:01.285116548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:07:02.907818058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T19:07:02.909442269Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T19:07:23.424184858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35980"
ts=2026-02-16T19:07:31.288007728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:07:46.096906606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:08:01.291575038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T19:08:13.483649149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:32768"
ts=2026-02-16T19:08:23.428890281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35818"
ts=2026-02-16T19:08:24.927987335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55476"
ts=2026-02-16T19:08:31.295112049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
level=debug ts=2026-02-16T19:08:44.279209014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:09:01.298016519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-xg7mt-29464e37 10.128.0.55:7946"
ts=2026-02-16T19:09:24.93293889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49192"
ts=2026-02-16T19:09:31.301035491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:09:46.824759751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T19:09:53.434294008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33662"
ts=2026-02-16T19:10:01.303964013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
ts=2026-02-16T19:10:13.491257252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48982"
ts=2026-02-16T19:10:31.306818253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e00f8275 10.128.0.58:7946"
level=debug ts=2026-02-16T19:10:47.878139872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"