level=info ts=2026-05-05T08:38:05.137271608Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-05T08:38:05.137311622Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-05T08:38:05.138793948Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-05T08:38:05.139226893Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T08:38:05.139329549Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.83 inf=eth0
level=debug ts=2026-05-05T08:38:05.140408221Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T08:38:05.140489413Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.83 inf=eth0
level=info ts=2026-05-05T08:38:05.140505262Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-01df19d2
ts=2026-05-05T08:38:05.140613101Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-05T08:38:05.140629644Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.83 advertisePort=7946
level=debug ts=2026-05-05T08:38:05.140643857Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.83 advertisePort=7946
level=info ts=2026-05-05T08:38:05.140779245Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-05T08:38:05.146663435Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T08:38:05.146683736Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T08:38:05.146697237Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.918953ms
level=error ts=2026-05-05T08:38:05.146709659Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-05T08:38:05.146719738Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-05T08:38:05.150497801Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T08:38:05.150519464Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T08:38:05.159648245Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-05T08:38:05.159644031Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-05T08:38:05.159670655Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-05T08:38:05.159683379Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-05T08:38:05.159686735Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-05T08:38:05.15976136Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-05T08:38:05.159798095Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-05T08:38:05.159813078Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-05T08:38:05.15989531Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-05T08:38:05.159899493Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-05T08:38:05.159953103Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-05T08:38:05.159967789Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-05T08:38:05.159982334Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-05T08:38:05.160160783Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-05T08:38:05.160174175Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-05T08:38:05.16019356Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-05T08:38:06.161119289Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-05T08:38:06.262371495Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-05T08:38:06.262426215Z caller=loki.go:615 msg="Loki started" startup_time=1.203995307s
ts=2026-05-05T08:38:06.775129274Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T08:38:06.775154552Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:38:09.289187689Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T08:38:09.289205811Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T08:38:11.262638086Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-05T08:38:11.262688067Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-05T08:38:14.666965962Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.55:7946: dial tcp 10.129.0.55:7946: connect: connection refused"
ts=2026-05-05T08:38:14.66866044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.86:7946"
level=info ts=2026-05-05T08:38:14.67053504Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=9.523809127s
level=debug ts=2026-05-05T08:38:24.590949684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:38:38.42990507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43520"
level=debug ts=2026-05-05T08:38:42.124306323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:38:55.007485286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:38:58.922296638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:45920"
level=debug ts=2026-05-05T08:39:07.340931682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:39:09.253451676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54914"
ts=2026-05-05T08:39:25.00864912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:39:28.296408317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:39:38.433932241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43772"
ts=2026-05-05T08:39:39.255470487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36292"
level=debug ts=2026-05-05T08:39:53.515419064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T08:39:53.517187639Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:39:55.010488721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:40:25.012497352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:40:29.086992123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:40:55.01414936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:40:58.933477815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42126"
level=debug ts=2026-05-05T08:41:18.428143528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:41:25.015922725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:41:39.264422054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45754"
ts=2026-05-05T08:41:55.018100922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:41:58.939438533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42496"
ts=2026-05-05T08:42:09.265972629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44144"
ts=2026-05-05T08:42:25.020085309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T08:42:28.889309845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:42:55.022540272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:43:09.268743842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57036"
ts=2026-05-05T08:43:25.024365443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:43:38.200999934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T08:43:38.452038223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:56074"
ts=2026-05-05T08:43:39.270412443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55198"
ts=2026-05-05T08:43:55.025217416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:44:25.027553321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:44:28.954024958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59126"
level=debug ts=2026-05-05T08:44:32.045893532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T08:44:32.048072875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:44:55.030658432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:45:09.276327485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38726"
level=debug ts=2026-05-05T08:45:18.542932812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:45:25.033928352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:45:38.458352756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49982"
ts=2026-05-05T08:45:55.035491672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:45:58.9629969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:43760"
ts=2026-05-05T08:46:08.459270528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41692"
level=debug ts=2026-05-05T08:46:10.648597523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:46:25.037334586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:46:28.966250695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57610"
level=debug ts=2026-05-05T08:46:53.110854506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:46:55.040904778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:47:25.042716674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:47:28.972778538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:54726"
level=debug ts=2026-05-05T08:47:52.97460322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:47:55.045249105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
level=info ts=2026-05-05T08:48:11.262698728Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-05T08:48:11.266336948Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:48:11.268214044Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:48:11.270277444Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:48:11.270332799Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T08:48:11.270344491Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-05T08:48:25.047323448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:48:29.92658591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T08:48:29.928626251Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:48:39.290080433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54884"
ts=2026-05-05T08:48:55.048950218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:48:58.981671352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44754"
ts=2026-05-05T08:49:09.291144559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39030"
ts=2026-05-05T08:49:25.050336114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T08:49:25.120870886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:49:28.984157712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:41868"
ts=2026-05-05T08:49:55.051414214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:50:14.226618508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:50:25.053247532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:50:38.476904923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54662"
ts=2026-05-05T08:50:55.055496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T08:50:55.568059395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:50:59.000448091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:45020"
ts=2026-05-05T08:51:08.478788266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53962"
ts=2026-05-05T08:51:25.056964908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:51:38.479955858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45624"
level=debug ts=2026-05-05T08:51:45.540026969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:51:55.058396068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:52:09.303960685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44846"
level=debug ts=2026-05-05T08:52:20.47362099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T08:52:20.475465532Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:52:25.060033729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:52:39.305825358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41080"
ts=2026-05-05T08:52:55.061715142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:53:09.307594628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52454"
level=debug ts=2026-05-05T08:53:15.091744896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:53:25.064830977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:53:29.015992335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51862"
ts=2026-05-05T08:53:55.067831263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
level=debug ts=2026-05-05T08:54:02.63042837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:54:09.310740212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52990"
ts=2026-05-05T08:54:25.070487367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:54:29.023201506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59650"
level=debug ts=2026-05-05T08:54:53.64761428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:54:55.072315712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:55:08.491639004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37606"
ts=2026-05-05T08:55:25.073398875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T08:55:29.040338508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:45302"
level=debug ts=2026-05-05T08:55:34.88422059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:55:39.323302899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60956"
ts=2026-05-05T08:55:55.074862163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:55:59.043011821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:55812"
ts=2026-05-05T08:56:08.496491599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36950"
level=debug ts=2026-05-05T08:56:20.448717773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T08:56:20.450597305Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:56:25.076598623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:56:29.046692525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:40504"
ts=2026-05-05T08:56:55.077608493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:57:08.500960633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41930"
level=debug ts=2026-05-05T08:57:19.159999586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:57:25.079655044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:57:38.502274057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:44882"
ts=2026-05-05T08:57:39.334826283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47736"
ts=2026-05-05T08:57:55.081516389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:58:08.50410688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33296"
level=debug ts=2026-05-05T08:58:09.558680414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T08:58:11.273426366Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:58:11.275229697Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:58:11.277579397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T08:58:11.277605973Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T08:58:25.082755304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T08:58:39.339705153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44088"
level=debug ts=2026-05-05T08:58:44.709001068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:58:55.085778803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T08:58:59.062740946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:39018"
ts=2026-05-05T08:59:25.089695895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T08:59:33.330909032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T08:59:55.091486565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:00:08.512878133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46538"
level=debug ts=2026-05-05T09:00:11.96908055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:00:11.970957338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:00:25.093959902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:00:29.073592808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38832"
ts=2026-05-05T09:00:38.514665463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39368"
ts=2026-05-05T09:00:55.095879779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:01:09.364763323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37016"
level=debug ts=2026-05-05T09:01:12.548567519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:01:25.097286793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:01:38.518859389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57734"
level=debug ts=2026-05-05T09:01:55.075393618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:01:55.098631027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:01:59.083399941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:35326"
ts=2026-05-05T09:02:08.519935135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53588"
ts=2026-05-05T09:02:09.368024102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47332"
ts=2026-05-05T09:02:25.100083292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:02:29.086599197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:51256"
level=debug ts=2026-05-05T09:02:54.714674695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:02:55.101194506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:03:09.372864336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36498"
ts=2026-05-05T09:03:25.102355436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:03:38.525644609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46390"
level=debug ts=2026-05-05T09:03:49.869575659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:03:55.103403988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:04:08.526625525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57428"
ts=2026-05-05T09:04:09.375130498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57460"
ts=2026-05-05T09:04:25.105182856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T09:04:36.806872747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:04:36.808755571Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:04:38.528472476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48034"
ts=2026-05-05T09:04:55.106364818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:04:59.101887424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59172"
ts=2026-05-05T09:05:08.530007053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50534"
ts=2026-05-05T09:05:09.380592845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42420"
ts=2026-05-05T09:05:25.107928822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:05:26.859201201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:05:38.530872358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59610"
ts=2026-05-05T09:05:55.110288688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
level=debug ts=2026-05-05T09:06:24.97821017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:06:25.11274822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:06:29.110709772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42252"
ts=2026-05-05T09:06:55.115235713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:07:09.387183664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46176"
level=debug ts=2026-05-05T09:07:21.942891997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:07:25.118685432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:07:29.116865208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:42326"
ts=2026-05-05T09:07:55.120985334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:08:08.543118655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58610"
ts=2026-05-05T09:08:09.391572929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39958"
level=error ts=2026-05-05T09:08:11.275158059Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:08:11.277461291Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:08:11.279750823Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:08:11.279774903Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T09:08:23.14610958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:08:25.122446626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:08:29.123383078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44818"
ts=2026-05-05T09:08:55.123934016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:09:14.559121662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:09:14.560832247Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:09:25.125445867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:09:55.12810708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:09:59.13337226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:52386"
level=debug ts=2026-05-05T09:10:13.57587542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:10:25.130408567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:10:29.136879348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:33234"
ts=2026-05-05T09:10:38.55267322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46984"
ts=2026-05-05T09:10:55.132079758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:11:09.404248994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35698"
level=debug ts=2026-05-05T09:11:12.058741129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:11:25.133430757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T09:11:50.609152008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:11:55.134404018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:12:09.409296641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49278"
ts=2026-05-05T09:12:25.135318171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:12:29.163530521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59160"
level=debug ts=2026-05-05T09:12:45.036692042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:12:55.137038893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:12:59.168740992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:46628"
ts=2026-05-05T09:13:25.138512158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:13:25.856151302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:13:25.857997647Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:13:39.415041148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34292"
ts=2026-05-05T09:13:55.140827041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
level=debug ts=2026-05-05T09:14:15.243135645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:14:25.142809405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:14:38.569979205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37752"
ts=2026-05-05T09:14:39.418283507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52058"
ts=2026-05-05T09:14:55.145262847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:14:59.180299518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:34792"
ts=2026-05-05T09:15:09.419407718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48824"
level=debug ts=2026-05-05T09:15:12.529703121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:15:25.147532754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:15:39.420957283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36782"
level=debug ts=2026-05-05T09:15:51.301871955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:15:55.149518442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:15:59.186312455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:53676"
ts=2026-05-05T09:16:09.422073153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51412"
ts=2026-05-05T09:16:25.150743823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
level=debug ts=2026-05-05T09:16:26.169838216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:16:38.577995551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58366"
ts=2026-05-05T09:16:39.423557446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60428"
ts=2026-05-05T09:16:55.15241253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:17:04.139006887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:17:04.141013745Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:17:08.579865843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49644"
ts=2026-05-05T09:17:25.153655849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:17:55.154645737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:17:59.199339297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38152"
level=debug ts=2026-05-05T09:18:04.017673267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:18:09.431616636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52396"
level=error ts=2026-05-05T09:18:11.273495167Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:18:11.275151192Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:18:11.276819449Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-05T09:18:11.276848764Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:18:25.155929411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:18:38.585531214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58158"
level=debug ts=2026-05-05T09:18:40.257723306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:18:55.156835539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:18:59.205011324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:41032"
level=debug ts=2026-05-05T09:19:22.726544852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:19:25.157869886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:19:29.207556452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:36804"
ts=2026-05-05T09:19:38.588083498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42140"
ts=2026-05-05T09:19:55.158951617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:20:06.756038882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:20:09.440390779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38184"
ts=2026-05-05T09:20:25.160867203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:20:39.442157984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37586"
ts=2026-05-05T09:20:55.163342612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:21:07.494346325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:21:07.495841476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:21:08.593910087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37180"
ts=2026-05-05T09:21:09.443983072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47456"
ts=2026-05-05T09:21:25.165099907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:21:29.219871362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:44560"
ts=2026-05-05T09:21:55.168192957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:21:59.223301375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:38288"
level=debug ts=2026-05-05T09:22:09.54382944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:22:25.170385254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:22:29.226452425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:59656"
ts=2026-05-05T09:22:38.598279514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54658"
level=debug ts=2026-05-05T09:22:49.38918952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:22:55.171259724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:22:59.229477101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:41970"
ts=2026-05-05T09:23:08.599735158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58814"
ts=2026-05-05T09:23:25.172349103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:23:29.232129993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48334"
level=debug ts=2026-05-05T09:23:34.169891773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:23:38.601442728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57192"
ts=2026-05-05T09:23:39.45454574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50970"
ts=2026-05-05T09:23:55.174685013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
ts=2026-05-05T09:23:59.235111218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:48764"
level=debug ts=2026-05-05T09:24:08.384130515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:24:08.60269941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53418"
ts=2026-05-05T09:24:09.456293199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37268"
ts=2026-05-05T09:24:25.176648942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:24:38.60411197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:60886"
level=debug ts=2026-05-05T09:24:45.976446339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T09:24:45.978061688Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-05T09:24:55.17741891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:24:59.241027845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:37774"
ts=2026-05-05T09:25:08.605393348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:44614"
ts=2026-05-05T09:25:25.178638314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
ts=2026-05-05T09:25:29.244457865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.86:57418"
ts=2026-05-05T09:25:38.607123043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51706"
level=debug ts=2026-05-05T09:25:39.685603849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:25:55.180199912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:26:25.181972635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-l7v7q-38bad6b9 10.130.0.86:7946"
level=debug ts=2026-05-05T09:26:26.041832345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T09:26:55.184251908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-mlwng-f127bf44 10.128.0.79:7946"
ts=2026-05-05T09:27:08.614873221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34912"
ts=2026-05-05T09:27:25.185809963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d1141cec 10.128.0.82:7946"
level=debug ts=2026-05-05T09:27:25.858133195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"