level=info ts=2026-05-07T05:24:18.114371784Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T05:24:18.114418908Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-07T05:24:18.116734747Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T05:24:18.117115308Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T05:24:18.117189982Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=debug ts=2026-05-07T05:24:18.117560338Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T05:24:18.117614258Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=info ts=2026-05-07T05:24:18.117649336Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f4ad714b
ts=2026-05-07T05:24:18.117708595Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T05:24:18.117720338Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=debug ts=2026-05-07T05:24:18.117746816Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=info ts=2026-05-07T05:24:18.117852672Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-07T05:24:18.122843595Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T05:24:18.122867715Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T05:24:18.122880148Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.028925ms
level=error ts=2026-05-07T05:24:18.122891224Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T05:24:18.122906484Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T05:24:18.125297691Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T05:24:18.125312499Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T05:24:18.132993369Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-07T05:24:18.132992771Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-07T05:24:18.133026193Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-07T05:24:18.133036953Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-07T05:24:18.133073258Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-07T05:24:18.133056055Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T05:24:18.133157244Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-07T05:24:18.133181084Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-07T05:24:18.133426756Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-07T05:24:18.133438325Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-07T05:24:18.133483665Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-07T05:24:18.133476636Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-07T05:24:18.133502934Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T05:24:18.133665597Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-07T05:24:18.13367895Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-07T05:24:18.133694024Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-07T05:24:19.133789272Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-07T05:24:19.240061597Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-07T05:24:19.240128429Z caller=loki.go:615 msg="Loki started" startup_time=1.189863598s
ts=2026-05-07T05:24:19.93809484Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T05:24:19.938122649Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:24:23.897854175Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T05:24:23.897878322Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T05:24:24.24099147Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-07T05:24:24.241061242Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-07T05:24:28.793882904Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T05:24:28.793905145Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:24:41.723518972Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
ts=2026-05-07T05:24:41.724290096Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-05-07T05:24:41.725029024Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
level=warn ts=2026-05-07T05:24:41.725041223Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="3 errors occurred:\n\t* failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused\n\t* failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused\n\t* failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused\n\n"
level=debug ts=2026-05-07T05:24:42.894409257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:24:45.648302032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:42408"
ts=2026-05-07T05:24:48.645681797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:24:51.237610643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:42458"
ts=2026-05-07T05:25:00.321349246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43706"
ts=2026-05-07T05:25:00.745794373Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
ts=2026-05-07T05:25:00.745917285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-05-07T05:25:00.745982842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36664"
ts=2026-05-07T05:25:00.74693862Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-05-07T05:25:00.747666291Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
ts=2026-05-07T05:25:00.748638695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-05-07T05:25:00.750058097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.84:7946"
ts=2026-05-07T05:25:00.751311294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.85:7946"
level=info ts=2026-05-07T05:25:00.752293178Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=42.629379611s
ts=2026-05-07T05:25:05.447972753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36934"
ts=2026-05-07T05:25:05.712302002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36940"
ts=2026-05-07T05:25:05.87622386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37142"
level=debug ts=2026-05-07T05:25:06.351544182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:25:18.647664881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:25:26.463152543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:25:30.32376798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33422"
ts=2026-05-07T05:25:48.649408963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:25:49.83111793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:25:51.241544822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:51816"
ts=2026-05-07T05:26:00.326222499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57166"
level=debug ts=2026-05-07T05:26:18.243266858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T05:26:18.244929151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:26:18.651000155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:26:35.71750486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:49498"
ts=2026-05-07T05:26:48.652558783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:26:50.291619128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:27:18.653908847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:27:48.655662834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:27:51.249361606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:51486"
level=debug ts=2026-05-07T05:28:00.969710321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:28:18.65827048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:28:21.251369498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:57014"
ts=2026-05-07T05:28:48.660708296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:28:51.253742602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:37260"
level=debug ts=2026-05-07T05:29:13.691315164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:29:18.662732785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:29:30.342349574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49838"
ts=2026-05-07T05:29:48.665442964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:30:00.344519743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59226"
level=debug ts=2026-05-07T05:30:16.628040117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-07T05:30:18.667456857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:30:30.346097696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40100"
ts=2026-05-07T05:30:35.735707503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45692"
ts=2026-05-07T05:30:48.669092267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:31:00.348562118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60594"
ts=2026-05-07T05:31:18.671346544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:31:24.345790354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T05:31:24.347330254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:31:48.673119297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T05:32:16.86664558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:32:18.675497583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:32:48.677420974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:33:04.10839621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:33:05.745808594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55682"
ts=2026-05-07T05:33:18.678871601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:33:48.680896606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:34:01.43771286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:34:05.748648992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57550"
ts=2026-05-07T05:34:18.682641414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:34:21.272759515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35422"
level=info ts=2026-05-07T05:34:24.241674665Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-07T05:34:24.24733518Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:34:24.249277523Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:34:24.251190869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:34:24.251213377Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T05:34:24.251225092Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-07T05:34:35.749943365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57406"
ts=2026-05-07T05:34:48.684628241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:35:01.453604693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:35:18.686267262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:35:43.774263791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:35:43.776131282Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:35:48.688703981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:36:00.368012425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54428"
ts=2026-05-07T05:36:18.689926464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:36:21.281339334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:56984"
level=debug ts=2026-05-07T05:36:21.330761263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:36:48.692242331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:36:51.283114952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:59766"
level=debug ts=2026-05-07T05:37:05.436381344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:37:18.694211749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:37:21.285136818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:52892"
ts=2026-05-07T05:37:48.695056213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:38:06.603454275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:38:18.695852653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:38:21.288835688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:47478"
ts=2026-05-07T05:38:48.698152426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:38:51.290891459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54346"
level=debug ts=2026-05-07T05:38:58.358964777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:39:00.378520339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43358"
ts=2026-05-07T05:39:18.700463738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:39:30.380644398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41478"
level=debug ts=2026-05-07T05:39:42.435993931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:39:42.437536514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:39:48.70299313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:40:05.772769079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46182"
ts=2026-05-07T05:40:18.705023387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T05:40:35.445717383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:40:48.707583637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:41:05.776881733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47046"
level=debug ts=2026-05-07T05:41:10.848916124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:41:18.709498321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:41:21.300371131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:37648"
ts=2026-05-07T05:41:30.387370852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44342"
ts=2026-05-07T05:41:48.712148598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:41:51.302893526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:48336"
level=debug ts=2026-05-07T05:41:52.809864042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:42:00.389654584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37144"
ts=2026-05-07T05:42:18.714883226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:42:48.716294314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T05:42:53.454592508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:43:18.71837767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:43:35.787252402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52394"
level=debug ts=2026-05-07T05:43:43.52602634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:43:43.527610652Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:43:48.720677008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:44:00.396685663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36854"
ts=2026-05-07T05:44:05.788346933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60390"
ts=2026-05-07T05:44:18.723230419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=error ts=2026-05-07T05:44:24.25625892Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:44:24.257977095Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:44:24.260649392Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:44:24.260672628Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T05:44:40.158878381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:44:48.725764967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:44:51.311602579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:53670"
ts=2026-05-07T05:45:18.728334241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:45:26.793132259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:45:48.730557851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:46:05.796638388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54434"
level=debug ts=2026-05-07T05:46:14.90730559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:46:18.73186258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:46:21.317325975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:56792"
ts=2026-05-07T05:46:30.405488957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46108"
ts=2026-05-07T05:46:48.733460111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:46:51.31986975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:37004"
level=debug ts=2026-05-07T05:47:13.133945439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:47:18.734860795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:47:30.40913049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41666"
ts=2026-05-07T05:47:35.802189308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37062"
ts=2026-05-07T05:47:48.735996416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:47:51.323406567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:39412"
level=debug ts=2026-05-07T05:47:54.959230647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:47:54.960629237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:48:18.737554372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:48:30.411738582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58358"
level=debug ts=2026-05-07T05:48:45.477629087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:48:48.739603235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:49:00.413454111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54220"
ts=2026-05-07T05:49:05.807616003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43146"
ts=2026-05-07T05:49:18.741385664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:49:34.156134375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:49:35.809444085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42944"
ts=2026-05-07T05:49:48.743269907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:50:00.416654039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42178"
ts=2026-05-07T05:50:05.811196776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52718"
ts=2026-05-07T05:50:18.7455053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:50:21.33343972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:43818"
level=debug ts=2026-05-07T05:50:31.992321691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:50:35.812007468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42552"
ts=2026-05-07T05:50:48.747234242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:50:51.336062139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46614"
level=debug ts=2026-05-07T05:51:12.725260425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:51:18.748538638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:51:30.423250953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36360"
ts=2026-05-07T05:51:35.815912694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35090"
ts=2026-05-07T05:51:48.750207922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:52:05.817538397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39018"
level=debug ts=2026-05-07T05:52:10.894608179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:52:10.896142883Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:52:18.7515408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:52:21.342353151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:36352"
ts=2026-05-07T05:52:48.753394535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:52:51.344354059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60446"
level=debug ts=2026-05-07T05:53:02.096856142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:53:05.820627426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:32786"
ts=2026-05-07T05:53:18.755194601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:53:21.3463258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:43874"
ts=2026-05-07T05:53:35.821799277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34448"
level=debug ts=2026-05-07T05:53:48.288392959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:53:48.757349268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:54:18.758996169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=error ts=2026-05-07T05:54:24.256829654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:54:24.258608996Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:54:24.26041847Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T05:54:24.260439005Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T05:54:28.209246738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:54:48.761085153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T05:55:15.051293331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:55:18.762307819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:55:48.764609527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:55:51.354097426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:39678"
level=debug ts=2026-05-07T05:55:54.010108938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:55:54.011697029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T05:56:18.766768221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T05:56:43.089115558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:56:48.768369062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T05:57:18.770762728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:57:21.36087926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:37904"
ts=2026-05-07T05:57:30.445135018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55250"
level=debug ts=2026-05-07T05:57:43.767507937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:57:48.773191096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:58:18.775415781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T05:58:21.365575545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:59696"
level=debug ts=2026-05-07T05:58:26.62473066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:58:30.448859709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53868"
ts=2026-05-07T05:58:35.847363737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36996"
ts=2026-05-07T05:58:48.77741802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:59:00.451173544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42690"
ts=2026-05-07T05:59:05.848300434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45352"
level=debug ts=2026-05-07T05:59:17.126884998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T05:59:18.779409198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T05:59:30.453783055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52208"
ts=2026-05-07T05:59:48.781779096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T05:59:52.304671305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T05:59:52.30657403Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:00:18.783545715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:00:21.371461427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:37994"
level=debug ts=2026-05-07T06:00:48.011271208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:00:48.785843339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:00:51.373263091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:42486"
ts=2026-05-07T06:01:05.856514112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56092"
ts=2026-05-07T06:01:18.788084769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:01:21.375257696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35584"
level=debug ts=2026-05-07T06:01:32.632060522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:01:35.85816675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54020"
ts=2026-05-07T06:01:48.790457383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:01:51.377472108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:50028"
ts=2026-05-07T06:02:05.859631554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47572"
ts=2026-05-07T06:02:18.792792285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:02:20.750173706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:02:48.794734856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:02:51.380925734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:59360"
ts=2026-05-07T06:03:00.467321579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60444"
level=debug ts=2026-05-07T06:03:04.980612755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:03:18.796415743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:03:21.382977427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54674"
ts=2026-05-07T06:03:30.469627598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41492"
ts=2026-05-07T06:03:35.865024554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55176"
ts=2026-05-07T06:03:48.798460369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:03:53.034892973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:03:53.036750126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:04:18.800046832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=error ts=2026-05-07T06:04:24.255295869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:04:24.257740004Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:04:24.259522757Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:04:24.259545653Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:04:30.473428648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36320"
ts=2026-05-07T06:04:35.869065226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58392"
level=debug ts=2026-05-07T06:04:39.653721032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:04:48.801739101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:04:51.388358788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:40008"
ts=2026-05-07T06:05:18.803752337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:05:35.873169614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:40580"
level=debug ts=2026-05-07T06:05:39.785112265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:05:48.805642124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:05:51.392452951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:42568"
ts=2026-05-07T06:06:18.807858074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:06:26.876146954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:06:35.876094268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47706"
ts=2026-05-07T06:06:48.810432197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:06:51.396849137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60356"
level=debug ts=2026-05-07T06:07:15.084843988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:07:18.813066943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:07:21.398547588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:45982"
ts=2026-05-07T06:07:48.8150096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:07:56.816864178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:07:56.818709136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:08:18.817439835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:08:35.88282659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39372"
level=debug ts=2026-05-07T06:08:45.996428274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:08:48.819688783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:08:51.403360241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:53894"
ts=2026-05-07T06:09:18.821486664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:09:21.406085605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38662"
level=debug ts=2026-05-07T06:09:33.954124738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:09:48.823278575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:10:05.888541643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:41088"
level=debug ts=2026-05-07T06:10:16.71110641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:10:18.825503064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:10:48.827534629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:11:01.70064897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:11:18.829645738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:11:21.413987363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46074"
ts=2026-05-07T06:11:30.496137143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49184"
ts=2026-05-07T06:11:48.831237611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T06:11:54.492522895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:11:54.494179308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:12:18.832761273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:12:21.418489333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:36612"
ts=2026-05-07T06:12:30.499363534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52922"
ts=2026-05-07T06:12:35.897644282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55622"
level=debug ts=2026-05-07T06:12:45.899755946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:12:48.834268243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:13:05.898970491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38634"
ts=2026-05-07T06:13:18.835854799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:13:21.423409286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38830"
ts=2026-05-07T06:13:30.503239501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54156"
level=debug ts=2026-05-07T06:13:40.251795893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:13:48.837372154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:14:00.505188155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58530"
ts=2026-05-07T06:14:18.839820419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=error ts=2026-05-07T06:14:24.254911589Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:14:24.256707465Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:14:24.258248761Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:14:24.258271536Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T06:14:34.878508411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:14:35.90453961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58046"
ts=2026-05-07T06:14:48.842588642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:14:51.430569067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:38060"
ts=2026-05-07T06:15:00.509712312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45762"
ts=2026-05-07T06:15:05.906277757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52098"
ts=2026-05-07T06:15:18.845311125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:15:23.790005689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:15:30.511669183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44628"
ts=2026-05-07T06:15:35.907788181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55390"
ts=2026-05-07T06:15:48.848314404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:16:04.532721799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:16:04.534270249Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:16:18.849894377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:16:35.911846083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53340"
ts=2026-05-07T06:16:48.85227764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:16:51.439669984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:39914"
level=debug ts=2026-05-07T06:16:55.285493693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:17:18.853835327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:17:30.520074712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55088"
ts=2026-05-07T06:17:48.85563408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:17:52.305517511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:18:18.858041092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:18:35.919247468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38136"
ts=2026-05-07T06:18:48.86078343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:18:53.699422176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:19:05.920784903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44964"
ts=2026-05-07T06:19:18.862510014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:19:21.449532269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:48114"
level=debug ts=2026-05-07T06:19:33.855622565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:19:48.864818258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:20:18.867452518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:20:21.453131881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:32818"
level=debug ts=2026-05-07T06:20:26.15732601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:20:26.159727967Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:20:48.869652344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:20:51.45512032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:41582"
ts=2026-05-07T06:21:05.929055634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43020"
level=debug ts=2026-05-07T06:21:15.586566764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:21:18.871652242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:21:35.929881131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37384"
ts=2026-05-07T06:21:48.873682555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:22:05.931471933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34670"
level=debug ts=2026-05-07T06:22:06.563048365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:22:18.875427491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:22:21.461672184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:51714"
ts=2026-05-07T06:22:35.933042146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34518"
ts=2026-05-07T06:22:48.87717929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:22:51.464202969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60148"
level=debug ts=2026-05-07T06:22:55.632336008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:23:18.87885519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:23:21.465986467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35194"
ts=2026-05-07T06:23:35.93700485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51218"
level=debug ts=2026-05-07T06:23:45.048189807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:23:48.880359454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:24:18.882489865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=error ts=2026-05-07T06:24:24.255669417Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:24:24.257492423Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:24:24.259029151Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:24:24.259055694Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:24:30.545551987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45342"
level=debug ts=2026-05-07T06:24:33.947231965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:24:33.948820325Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:24:48.884905483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:24:51.472304552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:48436"
ts=2026-05-07T06:25:18.887037765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T06:25:30.494841342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:25:30.548940023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52664"
ts=2026-05-07T06:25:35.94518034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53478"
ts=2026-05-07T06:25:48.888652679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:26:00.551134444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39730"
level=debug ts=2026-05-07T06:26:18.099610289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:26:18.890770566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:26:21.478548176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:49174"
ts=2026-05-07T06:26:30.553739704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37330"
ts=2026-05-07T06:26:48.892439057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:26:51.481169766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:59908"
ts=2026-05-07T06:27:00.555746361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57926"
level=debug ts=2026-05-07T06:27:16.078830636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:27:18.89453031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:27:21.483312528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46680"
ts=2026-05-07T06:27:30.558223354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43168"
ts=2026-05-07T06:27:48.896774222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:28:05.955061135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50490"
level=debug ts=2026-05-07T06:28:12.346446903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:28:18.898537022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:28:21.487483857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:39896"
ts=2026-05-07T06:28:35.95672103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53428"
ts=2026-05-07T06:28:48.900152633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:28:52.246889023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:28:52.248930256Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:29:05.95860153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50526"
ts=2026-05-07T06:29:18.902865188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:29:21.490413866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54214"
ts=2026-05-07T06:29:30.565844891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41486"
level=debug ts=2026-05-07T06:29:47.858471394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:29:48.905157972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:29:51.492359898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:40808"
ts=2026-05-07T06:30:00.568302675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50748"
ts=2026-05-07T06:30:18.907127286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:30:21.494009033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:34230"
level=debug ts=2026-05-07T06:30:22.162993608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:30:48.9097788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:31:00.341085046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:31:18.911894489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:31:48.913715358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T06:32:00.128566681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:32:18.916035024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:32:30.576823796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39794"
level=debug ts=2026-05-07T06:32:37.847286926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:32:37.849372091Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:32:48.918046843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:33:00.57949203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58848"
level=debug ts=2026-05-07T06:33:12.925397413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:33:18.919893779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:33:48.924214007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:33:51.505874374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:40784"
level=debug ts=2026-05-07T06:34:03.947394406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:34:05.981435666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44080"
ts=2026-05-07T06:34:18.926927749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=error ts=2026-05-07T06:34:24.255830988Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:34:24.257890914Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:34:24.259760478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:34:24.259784277Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:34:30.586353484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47128"
ts=2026-05-07T06:34:48.929213156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:34:51.510340277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:41760"
level=debug ts=2026-05-07T06:35:05.090287589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:35:18.930997481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:35:30.59073063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36230"
level=debug ts=2026-05-07T06:35:39.868927756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:35:48.933044315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:36:05.990068866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:33938"
ts=2026-05-07T06:36:18.935300536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:36:21.516187109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60128"
ts=2026-05-07T06:36:30.594265827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58754"
level=debug ts=2026-05-07T06:36:34.132399149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:36:34.134023569Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:36:48.937561543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:36:51.518805564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:45458"
ts=2026-05-07T06:37:00.596807757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:32992"
ts=2026-05-07T06:37:18.939769516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:37:21.476896864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:37:48.942586131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:37:51.523232674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:56900"
level=debug ts=2026-05-07T06:37:58.252441668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:38:18.94517485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:38:37.786759657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:38:48.947288996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:38:51.527939697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60426"
ts=2026-05-07T06:39:18.94927484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:39:24.285514284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:39:48.951513736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:40:15.144476298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:40:15.146121832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:40:18.954063751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:40:36.010389012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34682"
ts=2026-05-07T06:40:48.955519218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:40:51.536294353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:46784"
level=debug ts=2026-05-07T06:40:55.020191089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:41:18.957486924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:41:21.538662487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:44646"
ts=2026-05-07T06:41:30.610302528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50612"
ts=2026-05-07T06:41:36.014023811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57864"
level=debug ts=2026-05-07T06:41:42.605400625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:41:48.958649796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:42:18.960555337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T06:42:35.454661427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:42:48.962504433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:42:51.544774281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:40838"
ts=2026-05-07T06:43:06.019734302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38458"
ts=2026-05-07T06:43:18.965193412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:43:22.70924004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:43:36.020833932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45752"
ts=2026-05-07T06:43:48.967687697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:44:00.618788307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56476"
level=debug ts=2026-05-07T06:44:02.013399954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:44:02.01520371Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:44:18.970191696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=error ts=2026-05-07T06:44:24.255894598Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:44:24.257633821Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:44:24.25930796Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:44:24.259332328Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:44:36.024882751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56534"
level=debug ts=2026-05-07T06:44:46.72554562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:44:48.972415622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:44:51.55279817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54540"
ts=2026-05-07T06:45:00.622813435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37502"
ts=2026-05-07T06:45:06.026193617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55844"
ts=2026-05-07T06:45:18.97414351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:45:30.625481318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49162"
level=debug ts=2026-05-07T06:45:38.668876022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:45:48.975742132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:45:51.556289263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:36482"
ts=2026-05-07T06:46:06.029181834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58422"
level=debug ts=2026-05-07T06:46:18.180597727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:46:18.978033607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:46:21.55817584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:48572"
ts=2026-05-07T06:46:48.980513437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:46:51.560099804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:50438"
level=debug ts=2026-05-07T06:47:15.274770064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:47:18.982269387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:47:36.034136025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44836"
ts=2026-05-07T06:47:48.983456079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
level=debug ts=2026-05-07T06:47:51.327682696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:47:51.329542272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:47:51.564770141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:54568"
ts=2026-05-07T06:48:00.634895823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45708"
ts=2026-05-07T06:48:06.035660363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56128"
ts=2026-05-07T06:48:18.985730772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:48:21.566605896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:59688"
level=debug ts=2026-05-07T06:48:48.818242566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:48:48.988420475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:49:00.638448172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59374"
ts=2026-05-07T06:49:06.039039649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55568"
ts=2026-05-07T06:49:18.990598949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:49:21.571091446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60066"
level=debug ts=2026-05-07T06:49:32.668184252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:49:36.04000849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43504"
ts=2026-05-07T06:49:48.992741101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:50:06.041362063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51368"
ts=2026-05-07T06:50:18.995093126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:50:30.64471066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48870"
level=debug ts=2026-05-07T06:50:34.744601836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:50:48.997042753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:50:51.57837589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:56392"
ts=2026-05-07T06:51:18.998765061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:51:24.572538931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:51:49.001406968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:51:51.581861675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:57024"
ts=2026-05-07T06:52:19.003599842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:52:19.465698854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:52:19.467297238Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:52:30.652306867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42686"
ts=2026-05-07T06:52:49.006395936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:52:51.586415714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35016"
ts=2026-05-07T06:53:00.654748149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56118"
ts=2026-05-07T06:53:06.053243066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42186"
ts=2026-05-07T06:53:19.008048912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:53:20.111645843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:53:21.589031981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:35984"
ts=2026-05-07T06:53:49.010227814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"
ts=2026-05-07T06:53:51.590971558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:60758"
level=debug ts=2026-05-07T06:53:55.537548216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:54:19.01146626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:54:21.593582718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:51638"
level=error ts=2026-05-07T06:54:24.256034848Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:54:24.257814809Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:54:24.259811906Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-07T06:54:24.259833414Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:54:30.659995659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53322"
level=debug ts=2026-05-07T06:54:35.514002085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:54:36.058937012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60656"
ts=2026-05-07T06:54:49.01351974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:55:06.060247525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45694"
ts=2026-05-07T06:55:19.015833764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
level=debug ts=2026-05-07T06:55:26.348068415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:55:49.01814232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:55:51.599551513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:34322"
ts=2026-05-07T06:56:06.064079902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51990"
ts=2026-05-07T06:56:19.020386758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
level=debug ts=2026-05-07T06:56:21.014857693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T06:56:21.016537317Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-07T06:56:30.666140792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51166"
ts=2026-05-07T06:56:49.022573971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j9656-77245f54 10.129.0.84:7946"
ts=2026-05-07T06:57:00.667889773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57262"
ts=2026-05-07T06:57:06.067980121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51152"
level=debug ts=2026-05-07T06:57:09.490857899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:57:19.024378855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:57:21.60561019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.84:45290"
ts=2026-05-07T06:57:30.670540973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34582"
ts=2026-05-07T06:57:49.026302014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-r9wj5-2e7680ca 10.129.0.85:7946"
ts=2026-05-07T06:58:00.67297585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40470"
level=debug ts=2026-05-07T06:58:03.536223679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T06:58:06.072425362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58340"
ts=2026-05-07T06:58:19.028443737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-33de20be 10.130.0.73:7946"