level=info ts=2026-04-28T20:38:56.154429375Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-28T20:38:56.154467811Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-28T20:38:56.155712751Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T20:38:56.156157964Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T20:38:56.15625727Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
level=debug ts=2026-04-28T20:38:56.157748944Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T20:38:56.157837979Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
level=info ts=2026-04-28T20:38:56.157944063Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-6de2b4e5
ts=2026-04-28T20:38:56.158041022Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T20:38:56.158057666Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=debug ts=2026-04-28T20:38:56.158085556Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=info ts=2026-04-28T20:38:56.158252176Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T20:38:56.164718761Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T20:38:56.164735045Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:38:56.164744724Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.49582ms
level=error ts=2026-04-28T20:38:56.164754101Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-28T20:38:56.164765713Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-28T20:38:56.166996421Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:38:56.167012185Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T20:38:56.172613474Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-28T20:38:56.172676349Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-28T20:38:56.172711283Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-28T20:38:56.172769959Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-28T20:38:56.172781568Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-28T20:38:56.172792491Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T20:38:56.172871566Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-28T20:38:56.172891019Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-28T20:38:56.172907733Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-28T20:38:56.172917043Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-28T20:38:56.172976639Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-28T20:38:56.172982814Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-28T20:38:56.173006505Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T20:38:56.173155655Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-28T20:38:56.173166252Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-28T20:38:56.173180258Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-28T20:38:57.174296677Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-28T20:38:57.315658823Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-28T20:38:57.315709539Z caller=loki.go:615 msg="Loki started" startup_time=1.236272184s
ts=2026-04-28T20:38:57.73416761Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:38:57.734189552Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:39:00.581235049Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:39:00.581257579Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T20:39:02.316754219Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-28T20:39:02.316797304Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-28T20:39:04.838898546Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:39:04.838921923Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T20:39:15.994275843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:39:19.540933846Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused"
ts=2026-04-28T20:39:19.541504114Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.65:7946: dial tcp 10.130.0.65:7946: connect: connection refused"
ts=2026-04-28T20:39:19.542191912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-04-28T20:39:19.543646646Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.69:7946: dial tcp 10.130.0.69:7946: connect: connection refused"
ts=2026-04-28T20:39:19.543737623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
ts=2026-04-28T20:39:19.543768432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35652"
level=info ts=2026-04-28T20:39:19.543939288Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=23.379168985s
ts=2026-04-28T20:39:20.926790863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36560"
ts=2026-04-28T20:39:22.154108578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45476"
ts=2026-04-28T20:39:22.826398503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52846"
level=debug ts=2026-04-28T20:39:30.715252182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:39:40.045625239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:39:42.529050152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47008"
level=debug ts=2026-04-28T20:39:51.058517788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T20:40:07.166563361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:40:10.047628688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:40:12.531170693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53598"
level=debug ts=2026-04-28T20:40:37.575428235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T20:40:37.576944712Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:40:40.050274159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:40:45.473959786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35010"
ts=2026-04-28T20:41:10.052890922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T20:41:17.964032568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:41:40.054834838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:41:45.477055113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42596"
ts=2026-04-28T20:42:10.057181892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:42:12.538806626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47810"
level=debug ts=2026-04-28T20:42:14.401106231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:42:40.058681983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:43:10.059972466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:43:12.541762426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44166"
ts=2026-04-28T20:43:13.978528738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37188"
level=debug ts=2026-04-28T20:43:27.990688497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:43:40.06090223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:44:10.062881847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:44:13.981683743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45584"
ts=2026-04-28T20:44:15.486747576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41156"
level=debug ts=2026-04-28T20:44:31.707069235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:44:40.065842003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:44:42.547928786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49310"
ts=2026-04-28T20:44:43.983553776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59520"
ts=2026-04-28T20:44:45.488036868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41976"
ts=2026-04-28T20:45:10.068156044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:45:12.550317807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36052"
ts=2026-04-28T20:45:40.070188724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T20:45:41.081595492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T20:45:41.083204595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:45:42.551807134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37530"
ts=2026-04-28T20:45:45.49189428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47670"
ts=2026-04-28T20:46:10.071085114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:46:12.553778785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58974"
level=debug ts=2026-04-28T20:46:17.022031769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:46:40.072902658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:46:43.991383544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58340"
ts=2026-04-28T20:47:10.074963801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T20:47:11.441599938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:47:40.076728348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:47:42.558402046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59316"
ts=2026-04-28T20:47:43.994513359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53872"
ts=2026-04-28T20:47:45.499173054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50432"
level=debug ts=2026-04-28T20:48:06.192927675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:48:10.078393856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:48:40.079470744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:48:42.562400771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43782"
level=info ts=2026-04-28T20:49:02.316930822Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-28T20:49:02.31923926Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:49:02.321093273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:49:02.322677396Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:49:02.322701629Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T20:49:02.322713112Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-04-28T20:49:02.462797288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:49:10.080898378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:49:40.082674052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T20:49:55.534838115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:49:55.536325436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:50:10.084104997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:50:15.509661275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39754"
ts=2026-04-28T20:50:40.085982495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:50:42.569989216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52482"
ts=2026-04-28T20:50:45.511179933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44958"
level=debug ts=2026-04-28T20:50:53.722467413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:51:10.088253622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:51:14.006733813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60146"
ts=2026-04-28T20:51:15.51281124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49404"
ts=2026-04-28T20:51:40.090381556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:51:44.008775427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50972"
level=debug ts=2026-04-28T20:51:46.844530023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:52:10.09223326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T20:52:12.576417589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34030"
ts=2026-04-28T20:52:14.011483813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45904"
ts=2026-04-28T20:52:15.516806182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50208"
ts=2026-04-28T20:52:40.094370245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T20:52:40.159143713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:53:10.096048319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:53:14.015975583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54340"
ts=2026-04-28T20:53:15.520360952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55600"
ts=2026-04-28T20:53:40.097477477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T20:53:40.227252551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:54:10.098374182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:54:15.523544769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51762"
level=debug ts=2026-04-28T20:54:18.64601953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:54:18.647564927Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:54:40.099193615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:54:45.524831486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48244"
ts=2026-04-28T20:55:10.100570307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:55:12.589809559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49030"
level=debug ts=2026-04-28T20:55:16.781189543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:55:40.101708373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:55:42.592226019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51088"
ts=2026-04-28T20:55:45.528325013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47550"
ts=2026-04-28T20:56:10.102883327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T20:56:10.539642443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:56:40.104496819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T20:57:01.803768682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:57:10.106526776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T20:57:40.10755018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T20:57:49.248595631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:58:10.109222019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:58:14.03475477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47860"
level=debug ts=2026-04-28T20:58:25.047421807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:58:25.04945303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:58:40.112073003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:58:42.602894161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57368"
ts=2026-04-28T20:58:45.540963888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39312"
level=error ts=2026-04-28T20:59:02.327467932Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:59:02.330206781Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:59:02.33270408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T20:59:02.33272758Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:59:10.11408978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T20:59:17.741537996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:59:40.116316725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T20:59:42.607726511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48588"
ts=2026-04-28T20:59:44.040839674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41760"
level=debug ts=2026-04-28T21:00:01.083314622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:00:10.118828336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:00:40.120632018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:00:42.611634134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52146"
ts=2026-04-28T21:00:45.550081395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58946"
level=debug ts=2026-04-28T21:01:00.541441532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:01:10.123321121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:01:37.974946014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:01:40.124770773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:01:45.553709026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39030"
ts=2026-04-28T21:02:10.126542043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:02:21.092709045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:02:21.094413315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:02:40.128699111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:03:01.568688251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:03:10.130533119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:03:12.62210851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57350"
ts=2026-04-28T21:03:14.054457628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37766"
ts=2026-04-28T21:03:40.132652234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:03:45.562205817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58300"
level=debug ts=2026-04-28T21:03:47.351717885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:04:10.135052325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:04:15.564115406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34888"
level=debug ts=2026-04-28T21:04:27.997989899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:04:40.136961758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:04:42.628001974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59190"
level=debug ts=2026-04-28T21:05:02.326687937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:05:10.138831202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:05:12.630755202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44626"
ts=2026-04-28T21:05:40.140443058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:05:45.570664201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56098"
level=debug ts=2026-04-28T21:06:01.863641462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:06:01.86546682Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:06:10.142600169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:06:12.634896681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39542"
ts=2026-04-28T21:06:14.068610221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51030"
ts=2026-04-28T21:06:40.144656136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:06:45.582731422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48516"
level=debug ts=2026-04-28T21:07:00.198364763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:07:10.147325897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:07:14.072274545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54706"
ts=2026-04-28T21:07:15.585260273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51958"
level=debug ts=2026-04-28T21:07:37.581543472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:07:40.149807987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:08:10.154782233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:08:23.116630038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:08:40.156965786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=error ts=2026-04-28T21:09:02.327665483Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:09:02.32949993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:09:02.331303826Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:09:02.331328798Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:09:02.450399093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:09:10.158617933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:09:14.080787417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55198"
ts=2026-04-28T21:09:15.593238134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40142"
ts=2026-04-28T21:09:40.160923316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:09:42.661807361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40378"
ts=2026-04-28T21:09:45.594492422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34016"
level=debug ts=2026-04-28T21:10:01.854299474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:10:01.856248118Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:10:10.163433985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:10:12.664507558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59806"
ts=2026-04-28T21:10:40.165828834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:10:52.131371401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:11:10.167995067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:11:12.668843345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56338"
ts=2026-04-28T21:11:40.170237063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:11:44.092763008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44416"
level=debug ts=2026-04-28T21:11:53.453213277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:12:10.172056243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:12:14.095415119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40402"
ts=2026-04-28T21:12:15.607555896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36368"
level=debug ts=2026-04-28T21:12:38.828724531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:12:40.173092789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:12:42.674128081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60570"
ts=2026-04-28T21:13:10.174565498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:13:14.099421781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49304"
ts=2026-04-28T21:13:15.611563469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33472"
level=debug ts=2026-04-28T21:13:32.756173348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:13:40.176197948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:14:10.17848702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:14:15.614392096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59302"
level=debug ts=2026-04-28T21:14:33.13691849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:14:33.138678374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:14:40.180703894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:15:10.182727257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:15:12.683829006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40512"
ts=2026-04-28T21:15:15.618106797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60066"
level=debug ts=2026-04-28T21:15:20.572977094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:15:40.18692868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:15:45.619701954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48498"
ts=2026-04-28T21:16:10.189995302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:16:12.946461539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:16:15.621079188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48752"
ts=2026-04-28T21:16:40.193980166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:17:03.332613593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:17:10.200694283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:17:15.624350148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36114"
ts=2026-04-28T21:17:40.204430571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:17:42.698750551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37008"
ts=2026-04-28T21:17:44.118466606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51552"
level=debug ts=2026-04-28T21:17:48.901990758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:18:10.207052112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:18:40.209187322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:18:40.94917297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:18:40.950710792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:18:42.703062223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36148"
level=error ts=2026-04-28T21:19:02.329142034Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:19:02.33136518Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:19:02.333284561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:19:02.333314003Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:19:10.210997124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:19:15.633469247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36272"
level=debug ts=2026-04-28T21:19:16.31462804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:19:40.212968393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:19:53.64013243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:20:10.215371275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:20:15.636481858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57066"
ts=2026-04-28T21:20:40.217613983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:20:40.711810982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:21:10.220230003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:21:12.712787746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58154"
ts=2026-04-28T21:21:14.133646343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49678"
level=debug ts=2026-04-28T21:21:27.718834407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:21:40.222007962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:21:44.136033656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51568"
ts=2026-04-28T21:22:10.223640412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:22:12.715739479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59658"
ts=2026-04-28T21:22:14.13811655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54946"
level=debug ts=2026-04-28T21:22:21.98916862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:22:21.990692283Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:22:40.224967578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:22:45.648520546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36110"
level=debug ts=2026-04-28T21:23:02.484781576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:23:10.226185934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:23:14.142901205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46110"
ts=2026-04-28T21:23:40.227860644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:23:45.652201224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47032"
level=debug ts=2026-04-28T21:23:53.425647478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:24:10.229845404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:24:31.517601527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:24:40.23223762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:24:44.149152359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42522"
ts=2026-04-28T21:25:10.234317576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:25:13.172001322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:25:15.660406931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42824"
ts=2026-04-28T21:25:40.236826999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:25:44.153310832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43136"
ts=2026-04-28T21:25:45.662173308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49966"
level=debug ts=2026-04-28T21:26:04.609167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:26:04.611290102Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:26:10.239097815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:26:12.730699029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41838"
ts=2026-04-28T21:26:40.241170142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:27:00.003929191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:27:10.243508254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:27:12.733939886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35276"
ts=2026-04-28T21:27:14.156992326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51954"
ts=2026-04-28T21:27:15.667234478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37254"
level=debug ts=2026-04-28T21:27:39.50342664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:27:40.2452988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:27:42.737409231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60674"
ts=2026-04-28T21:28:10.248046017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:28:12.74017063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56884"
level=debug ts=2026-04-28T21:28:30.015097144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:28:40.250228638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=error ts=2026-04-28T21:29:02.327362827Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:29:02.329077695Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:29:02.330701303Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:29:02.330729815Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:29:10.25201279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:29:14.165479006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35490"
level=debug ts=2026-04-28T21:29:17.620279914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:29:40.253139613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:29:44.168082609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55340"
ts=2026-04-28T21:30:10.254478689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:30:14.170247404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40812"
level=debug ts=2026-04-28T21:30:15.412436999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:30:15.413944965Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:30:15.680669684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58616"
ts=2026-04-28T21:30:40.255606977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:30:42.750058625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44938"
ts=2026-04-28T21:30:44.172893736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55126"
level=debug ts=2026-04-28T21:30:54.554382214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:31:10.257136509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:31:38.502282349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:31:40.259093933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:31:45.688325723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57854"
ts=2026-04-28T21:32:10.260929341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:32:14.178218389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49358"
level=debug ts=2026-04-28T21:32:27.121049092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:32:40.263299102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:32:45.694699243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51074"
ts=2026-04-28T21:33:10.265673013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:33:12.759359538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47656"
ts=2026-04-28T21:33:14.185921791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51056"
level=debug ts=2026-04-28T21:33:26.124979405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:33:40.268107631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:33:44.189716268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56030"
ts=2026-04-28T21:34:10.270138508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:34:12.763734956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47676"
level=debug ts=2026-04-28T21:34:22.428995853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:34:22.430568577Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:34:40.272333618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:34:42.765694697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40324"
ts=2026-04-28T21:34:44.19469307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56080"
ts=2026-04-28T21:34:45.70418662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48586"
level=debug ts=2026-04-28T21:34:59.241614331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:35:10.27471954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:35:15.705164788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58584"
ts=2026-04-28T21:35:40.276615133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:35:43.27301506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:35:45.706549062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60086"
ts=2026-04-28T21:36:10.278845923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:36:25.704709283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:36:40.281803711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:36:42.774519492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38588"
ts=2026-04-28T21:36:45.710357273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60948"
ts=2026-04-28T21:37:10.284188189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:37:14.204871451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41960"
level=debug ts=2026-04-28T21:37:14.94228968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:37:40.286668898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:37:42.779033079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45006"
ts=2026-04-28T21:37:44.207082414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33360"
level=debug ts=2026-04-28T21:37:59.708641779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:37:59.710182933Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:38:10.289106003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:38:12.781722244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45658"
ts=2026-04-28T21:38:40.291240144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:38:42.784325974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54016"
level=debug ts=2026-04-28T21:38:59.524570586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:39:02.32535112Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:39:02.326862843Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:39:02.328304689Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:39:02.328326058Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:39:10.294567148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:39:12.786814725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39960"
ts=2026-04-28T21:39:15.720300089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50084"
ts=2026-04-28T21:39:40.296077516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:39:44.21435194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52368"
level=debug ts=2026-04-28T21:39:50.395407787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:40:10.297680566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:40:12.791301747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41746"
ts=2026-04-28T21:40:14.216303625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58864"
ts=2026-04-28T21:40:40.299409099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:40:42.261914855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:40:42.792935714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47636"
ts=2026-04-28T21:41:10.302038069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:41:12.794995893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:55140"
ts=2026-04-28T21:41:40.303981561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:41:42.798545955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56260"
level=debug ts=2026-04-28T21:41:43.575299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:42:10.306690781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:42:14.224265258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39170"
level=debug ts=2026-04-28T21:42:31.229219247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:42:31.231510847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:42:40.308318567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:43:10.310034868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:43:12.804776742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58456"
ts=2026-04-28T21:43:15.736607934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39102"
level=debug ts=2026-04-28T21:43:27.37705087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:43:40.311522116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:43:42.806767948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36864"
ts=2026-04-28T21:43:45.73739691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55126"
ts=2026-04-28T21:44:10.313624456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:44:12.352760805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:44:15.738859786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49482"
ts=2026-04-28T21:44:40.316206106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:44:51.312088502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:45:10.318349951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:45:14.236980685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55318"
ts=2026-04-28T21:45:15.742664218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40890"
level=debug ts=2026-04-28T21:45:27.015158168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:45:40.320462021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:45:42.81450828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53888"
level=debug ts=2026-04-28T21:46:01.994505585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:46:01.996067345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:46:10.322384338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:46:14.241634572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38642"
ts=2026-04-28T21:46:40.324880023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:46:44.24349311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56384"
level=debug ts=2026-04-28T21:46:47.980193825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:47:10.326527288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:47:23.91424372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:47:40.328593193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:47:44.247935356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58822"
ts=2026-04-28T21:48:10.330744644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:48:11.285437495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:48:14.250356597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56690"
ts=2026-04-28T21:48:15.755826363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41112"
ts=2026-04-28T21:48:40.332774527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:48:42.824896158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40794"
level=error ts=2026-04-28T21:49:02.327732533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:49:02.329679868Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:49:02.331609098Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:49:02.331630812Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:49:07.175534637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:49:10.33464365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:49:14.253973979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50740"
ts=2026-04-28T21:49:15.759119616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57992"
ts=2026-04-28T21:49:40.336682852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:49:44.255962726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41454"
ts=2026-04-28T21:49:45.760601337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50864"
level=debug ts=2026-04-28T21:50:02.426750506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:50:02.428764631Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:50:10.338160233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:50:15.762295462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39572"
ts=2026-04-28T21:50:40.340272752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:50:42.832702291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37602"
level=debug ts=2026-04-28T21:50:53.909604893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:51:10.342770339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:51:15.765790466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38978"
ts=2026-04-28T21:51:40.344701766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:51:42.836863491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35348"
level=debug ts=2026-04-28T21:51:45.244684798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:52:10.346978851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:52:14.266613486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33250"
ts=2026-04-28T21:52:40.348456882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:52:41.713158316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:52:42.841699772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50140"
ts=2026-04-28T21:52:45.771549316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42074"
ts=2026-04-28T21:53:10.350401566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:53:12.843821966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58886"
ts=2026-04-28T21:53:14.269891122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59236"
ts=2026-04-28T21:53:15.772992908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37752"
level=debug ts=2026-04-28T21:53:19.654245981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:53:40.352537019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:54:10.355342382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:54:12.849600224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45752"
ts=2026-04-28T21:54:14.273173187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33388"
level=debug ts=2026-04-28T21:54:20.983396715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:54:20.985264704Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:54:40.357197541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T21:54:57.256189791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:55:10.358885441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T21:55:14.27792855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33460"
ts=2026-04-28T21:55:40.360483333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:55:44.269396737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:56:10.362708554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:56:14.282894259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43774"
ts=2026-04-28T21:56:15.794673059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41984"
level=debug ts=2026-04-28T21:56:33.389385972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:56:40.365036693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:56:45.796551357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38884"
ts=2026-04-28T21:57:10.3677875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:57:14.288009623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41758"
ts=2026-04-28T21:57:15.797547624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51048"
level=debug ts=2026-04-28T21:57:34.586782441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:57:40.369650799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:57:44.289956714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36876"
ts=2026-04-28T21:58:10.371389423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:58:15.801899111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44914"
level=debug ts=2026-04-28T21:58:21.141925316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:58:21.14349614Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:58:40.374088534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T21:58:44.293973025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57436"
ts=2026-04-28T21:58:45.803536926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43046"
level=error ts=2026-04-28T21:59:02.327304331Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:59:02.329320852Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:59:02.331139162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T21:59:02.331166482Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:59:10.376910449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:59:14.29579974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56006"
ts=2026-04-28T21:59:15.804845851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47226"
level=debug ts=2026-04-28T21:59:21.78010671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:59:40.379647479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T21:59:44.298209484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35932"
ts=2026-04-28T21:59:45.805992915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57762"
ts=2026-04-28T22:00:10.381716933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:00:12.874542281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56980"
level=debug ts=2026-04-28T22:00:23.334819812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:00:40.383470564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T22:00:58.127902478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:01:10.386001274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:01:12.877730052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:55094"
ts=2026-04-28T22:01:14.303825049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58068"
ts=2026-04-28T22:01:15.812715968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45518"
ts=2026-04-28T22:01:40.388592665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:01:42.879684084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45568"
level=debug ts=2026-04-28T22:01:44.808683229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:01:45.81386803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58864"
ts=2026-04-28T22:02:10.390879265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:02:14.308501892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44754"
ts=2026-04-28T22:02:40.392871138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:02:45.816816527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50452"
level=debug ts=2026-04-28T22:02:46.155804109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:02:46.157375163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:03:10.394454356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:03:12.887536392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41168"
ts=2026-04-28T22:03:14.31343148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53208"
level=debug ts=2026-04-28T22:03:39.178346629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:03:40.395767322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:03:45.820019174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41746"
ts=2026-04-28T22:04:10.397857202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:04:15.821144031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37730"
level=debug ts=2026-04-28T22:04:24.528592382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:04:40.399379217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:04:44.318947871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47112"
ts=2026-04-28T22:04:45.821949851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59284"
ts=2026-04-28T22:05:10.400689011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:05:14.320909931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46350"
ts=2026-04-28T22:05:15.823741431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37036"
level=debug ts=2026-04-28T22:05:16.158757911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:05:40.402180052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:05:44.323484686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33470"
ts=2026-04-28T22:06:10.404157722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:06:14.325883906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58060"
level=debug ts=2026-04-28T22:06:16.885322392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:06:40.405941645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T22:06:54.670641712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:06:54.672790019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:07:10.407659879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:07:12.905851266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53196"
ts=2026-04-28T22:07:15.830981408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54528"
ts=2026-04-28T22:07:40.40896293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T22:07:41.700786823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:07:42.908339426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48808"
ts=2026-04-28T22:07:45.831980788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34978"
ts=2026-04-28T22:08:10.410111649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:08:12.910939526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41968"
ts=2026-04-28T22:08:14.3344859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56410"
level=debug ts=2026-04-28T22:08:18.661810378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:08:40.411712216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:08:45.835877093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47364"
level=error ts=2026-04-28T22:09:02.326927965Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:09:02.328970481Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:09:02.330830283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:09:02.330853123Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:09:10.413421979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:09:14.33853647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41704"
ts=2026-04-28T22:09:15.837125159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55852"
level=debug ts=2026-04-28T22:09:18.236948402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:09:40.414773253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:10:10.416116423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T22:10:17.659740298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:10:40.417437996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:10:44.343640998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57948"
ts=2026-04-28T22:11:10.419088002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:11:14.34562866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54652"
level=debug ts=2026-04-28T22:11:19.306010814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:11:19.307449978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:11:40.420034547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:11:44.347317394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38122"
level=debug ts=2026-04-28T22:12:03.883634159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:12:10.421772679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:12:15.847295829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37072"
ts=2026-04-28T22:12:40.424356477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:12:42.93087069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:40202"
level=debug ts=2026-04-28T22:12:52.943743725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:13:10.426449673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:13:14.353225556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54002"
level=debug ts=2026-04-28T22:13:29.959032311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:13:40.428112857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:13:44.354900757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56370"
ts=2026-04-28T22:14:10.429075986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T22:14:13.872458635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:14:40.431391524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:14:45.858809254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54340"
level=debug ts=2026-04-28T22:15:10.038943359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:15:10.040624637Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:15:10.433055081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:15:12.942495483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52730"
ts=2026-04-28T22:15:40.435327832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T22:15:50.234365389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:16:10.437770365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:16:12.94650703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48774"
ts=2026-04-28T22:16:14.364819287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54432"
ts=2026-04-28T22:16:15.863872863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52932"
level=debug ts=2026-04-28T22:16:32.590567362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:16:40.440045276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
ts=2026-04-28T22:16:44.366980112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34290"
ts=2026-04-28T22:17:10.44211068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T22:17:29.298338161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:17:40.443538779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:18:10.444871533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:18:12.95427063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39668"
level=debug ts=2026-04-28T22:18:20.796054631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:18:40.447027002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T22:19:00.79986638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:19:00.801659706Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:19:02.325847738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:19:02.327455351Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:19:02.329671056Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T22:19:02.329695405Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:19:10.448330254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:19:40.449990803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T22:19:42.566299194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:19:42.96208127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:60132"
ts=2026-04-28T22:19:45.87908834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51384"
ts=2026-04-28T22:20:10.452024313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:20:12.964752442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56302"
ts=2026-04-28T22:20:15.880194705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33882"
level=debug ts=2026-04-28T22:20:23.149341657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:20:40.454367801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
level=debug ts=2026-04-28T22:21:00.757684011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:21:10.45735006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b169d989 10.128.0.88:7946"
ts=2026-04-28T22:21:15.883536752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42956"
ts=2026-04-28T22:21:40.459146576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
ts=2026-04-28T22:21:42.970357371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57350"
level=debug ts=2026-04-28T22:21:55.130535593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:22:10.460995703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"