level=info ts=2026-05-13T12:44:11.337564769Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T12:44:11.33785193Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-13T12:44:11.339170604Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T12:44:11.339647982Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:44:11.339763257Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.57 inf=eth0
level=debug ts=2026-05-13T12:44:11.34123563Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:44:11.341329476Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.57 inf=eth0
level=info ts=2026-05-13T12:44:11.34149839Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-177c7b43
ts=2026-05-13T12:44:11.341560594Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T12:44:11.341573215Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.57 advertisePort=7946
level=debug ts=2026-05-13T12:44:11.341593608Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.57 advertisePort=7946
level=info ts=2026-05-13T12:44:11.341736541Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-13T12:44:11.347853872Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:44:11.347881701Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:44:11.347904025Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.16881ms
level=error ts=2026-05-13T12:44:11.347916152Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T12:44:11.34792562Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-13T12:44:11.352208077Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:11.352231111Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T12:44:11.358823822Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T12:44:11.358849875Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-13T12:44:11.358901185Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T12:44:11.359163127Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-13T12:44:11.359193721Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-13T12:44:11.359202537Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T12:44:11.359315477Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-13T12:44:11.35932595Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T12:44:11.359467035Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-13T12:44:11.359483303Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-13T12:44:11.359541531Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-13T12:44:11.359588375Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-13T12:44:11.359612263Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T12:44:11.359774914Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-13T12:44:11.359794227Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-13T12:44:11.359885768Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-13T12:44:12.360313562Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-13T12:44:12.464502687Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-13T12:44:12.46454321Z caller=loki.go:615 msg="Loki started" startup_time=1.201967225s
ts=2026-05-13T12:44:13.321743415Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:13.321767589Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:44:16.097542318Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:16.097569149Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T12:44:17.465567474Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-13T12:44:17.465623465Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-13T12:44:21.752072788Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:21.752094764Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T12:44:22.952286904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:44:32.729599798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56216"
ts=2026-05-13T12:44:35.48648547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.55:7946"
ts=2026-05-13T12:44:35.487733923Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.56:7946: dial tcp 10.128.0.56:7946: connect: connection refused"
ts=2026-05-13T12:44:35.487817629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-05-13T12:44:35.487859438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43642"
ts=2026-05-13T12:44:35.488276739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.54:7946"
level=info ts=2026-05-13T12:44:35.488576404Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.140644592s
ts=2026-05-13T12:44:36.086716989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57100"
ts=2026-05-13T12:44:36.24712957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53194"
level=debug ts=2026-05-13T12:44:40.63634794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:44:43.946331507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59316"
ts=2026-05-13T12:44:46.066269911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:44:59.826985298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:45:13.310217635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58068"
ts=2026-05-13T12:45:16.068037259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:45:22.372970826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:45:43.311609514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42544"
ts=2026-05-13T12:45:46.069256983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:45:48.965161739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:45:48.966808365Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:46:13.31263609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41118"
ts=2026-05-13T12:46:16.071015612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:46:26.770662983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:46:43.313715967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59234"
ts=2026-05-13T12:46:46.072198913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:47:13.315272977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39290"
ts=2026-05-13T12:47:16.073256463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T12:47:30.261019774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59580"
level=debug ts=2026-05-13T12:47:32.75677762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:47:43.316588228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36566"
ts=2026-05-13T12:47:46.074149745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:48:00.261987021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37038"
ts=2026-05-13T12:48:13.317677835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56584"
ts=2026-05-13T12:48:13.962466168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51700"
ts=2026-05-13T12:48:16.075168411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:48:37.307129363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:48:46.076755237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:49:16.078221554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:49:30.265960729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59650"
ts=2026-05-13T12:49:46.079339037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:49:46.998845808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:50:16.080970038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:50:39.423730078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:50:39.425404905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:50:43.324125006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40902"
ts=2026-05-13T12:50:43.968919145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58324"
ts=2026-05-13T12:50:46.082205185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:51:16.083663391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:51:30.271064541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:47282"
level=debug ts=2026-05-13T12:51:36.258521963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:51:43.97128571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44924"
ts=2026-05-13T12:51:46.084780255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:52:11.910283209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:52:13.972309677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48398"
ts=2026-05-13T12:52:16.0858002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:52:46.086577722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:53:08.481746228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:53:16.088168592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:53:30.275483804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49916"
ts=2026-05-13T12:53:46.089836788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:53:55.756069296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:54:13.333599631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44848"
ts=2026-05-13T12:54:16.090807326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T12:54:17.466600372Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-13T12:54:17.470552942Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:54:17.472263707Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:54:17.473766992Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:54:17.473789537Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:54:17.473801302Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-13T12:54:37.14762207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:54:37.149477929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:54:46.092297395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:55:16.093486391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:55:26.147836509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:55:30.280925765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54644"
ts=2026-05-13T12:55:46.0951645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:56:00.282058982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54048"
level=debug ts=2026-05-13T12:56:05.294739801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:56:13.982520692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46470"
ts=2026-05-13T12:56:16.097157315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:56:30.283747111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:46324"
ts=2026-05-13T12:56:46.09836027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:56:55.365848188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:57:16.100097944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T12:57:43.98564456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:36654"
ts=2026-05-13T12:57:46.101321159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:57:47.371782778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:58:00.288403474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49442"
ts=2026-05-13T12:58:13.98692238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49260"
ts=2026-05-13T12:58:16.103156766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T12:58:42.028528276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:58:42.030630168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:58:43.988388982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51168"
ts=2026-05-13T12:58:46.105207427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:59:13.347522484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48582"
ts=2026-05-13T12:59:16.106222759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:59:29.657442786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:59:30.29637158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37972"
ts=2026-05-13T12:59:46.107783744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:00:13.352145206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56560"
ts=2026-05-13T13:00:16.109411424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:00:27.76056294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:00:30.299355625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59814"
ts=2026-05-13T13:00:46.110551924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:01:13.997083059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35656"
ts=2026-05-13T13:01:16.112296147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:01:24.590314931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:01:46.114006158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:02:00.304582516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37550"
ts=2026-05-13T13:02:16.115069584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:02:27.02793571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:02:46.116652764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:03:00.308205168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42360"
ts=2026-05-13T13:03:16.118228744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:03:29.047818167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:03:29.050106842Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:03:43.362247621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38222"
ts=2026-05-13T13:03:46.119943239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:04:03.857522638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:04:16.121702184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=error ts=2026-05-13T13:04:17.479382799Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:04:17.481499298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:04:17.483710173Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:04:17.483733634Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:04:41.159953628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:04:43.36507565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37706"
ts=2026-05-13T13:04:46.122991758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:05:00.313809609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43772"
ts=2026-05-13T13:05:16.124881778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:05:25.928097888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:05:44.011186239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40018"
ts=2026-05-13T13:05:46.125887061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:06:16.127863559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:06:20.557128052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:06:44.013811401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51168"
ts=2026-05-13T13:06:46.128959768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:07:13.557070579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:07:13.558865333Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:07:16.130914918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:07:44.016260957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56772"
ts=2026-05-13T13:07:46.132580707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:08:06.782980585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:08:14.017884393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49728"
ts=2026-05-13T13:08:16.133427147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:08:43.374873535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45306"
ts=2026-05-13T13:08:44.019148484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50448"
ts=2026-05-13T13:08:46.134624153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:08:48.970710905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:09:13.376402373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53278"
ts=2026-05-13T13:09:16.136341288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:09:41.742254894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:09:44.022568601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44278"
ts=2026-05-13T13:09:46.137169951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:10:16.13903719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:10:26.301656743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:10:43.381057753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57762"
ts=2026-05-13T13:10:44.026073631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46020"
ts=2026-05-13T13:10:46.141516879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:11:09.650000973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:11:09.652119208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:11:14.027985223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50264"
ts=2026-05-13T13:11:16.143612842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:11:43.382981535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46710"
ts=2026-05-13T13:11:44.029639779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35252"
ts=2026-05-13T13:11:46.145418422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:12:07.960625732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:12:16.147122829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:12:43.386207695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47064"
ts=2026-05-13T13:12:44.031757803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46906"
ts=2026-05-13T13:12:46.147977344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:13:01.800584863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:13:14.033412119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51636"
ts=2026-05-13T13:13:16.149772473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:13:43.388084392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59610"
ts=2026-05-13T13:13:46.151360905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:13:46.84329012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:14:16.153229779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=error ts=2026-05-13T13:14:17.47925139Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:14:17.481211782Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:14:17.483039299Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:14:17.48306173Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:14:38.34688361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:14:43.390730565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41844"
ts=2026-05-13T13:14:46.154378749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:15:16.155544077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:15:23.325875685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:15:23.327694198Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:15:43.393670748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54062"
ts=2026-05-13T13:15:46.156634756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:16:16.157911724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:16:16.507533618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:16:46.15918833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:17:15.83526964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:17:16.160394301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:17:43.398966969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44888"
ts=2026-05-13T13:17:46.162126301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:18:00.167016321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:18:13.399925607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57512"
ts=2026-05-13T13:18:14.047512474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44876"
ts=2026-05-13T13:18:16.163808456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:18:43.400927151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45260"
ts=2026-05-13T13:18:46.165153856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:18:46.24907115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:19:16.166598697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:19:31.143092628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:19:31.144817954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:19:43.403731004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57074"
ts=2026-05-13T13:19:46.168113112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:20:10.688097531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:20:16.169626236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:20:46.170750565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:20:46.460694069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:21:13.407989522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59078"
ts=2026-05-13T13:21:14.056264688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:37916"
ts=2026-05-13T13:21:16.171538578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:21:30.523065546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:21:44.057201498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47320"
ts=2026-05-13T13:21:46.173194149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:22:13.410352505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56008"
ts=2026-05-13T13:22:16.17443651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:22:23.559287962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:22:43.411201612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57418"
ts=2026-05-13T13:22:46.175372009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:23:09.542271001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:23:09.544116023Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:23:16.176931962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:23:43.413544076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40142"
ts=2026-05-13T13:23:46.178491011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:24:10.597932643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:24:16.180106331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=error ts=2026-05-13T13:24:17.47641875Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:24:17.478076556Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:24:17.47966268Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:24:17.479685115Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:24:30.367605302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54160"
ts=2026-05-13T13:24:44.065461226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55692"
ts=2026-05-13T13:24:46.181739537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:24:47.922089525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:25:13.418401838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55054"
ts=2026-05-13T13:25:16.182823517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:25:41.239791962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:25:46.184383403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:26:00.371523927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33350"
ts=2026-05-13T13:26:16.185916407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:26:17.470582854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:26:46.187039143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:27:00.374564467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41266"
ts=2026-05-13T13:27:13.424026478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46768"
level=debug ts=2026-05-13T13:27:14.515139054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:27:14.516742167Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:27:16.188284834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:27:46.189980933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:28:09.399003705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:28:13.427105231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48788"
ts=2026-05-13T13:28:16.191607278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:28:44.076303034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55672"
ts=2026-05-13T13:28:46.192538855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:28:54.845711422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:29:16.193624942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:29:30.381380323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:40846"
ts=2026-05-13T13:29:44.079333418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40286"
ts=2026-05-13T13:29:46.194611749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:29:56.77857898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:30:00.382745762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:32910"
ts=2026-05-13T13:30:14.080473606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52728"
ts=2026-05-13T13:30:16.195484129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:30:44.081731995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33894"
ts=2026-05-13T13:30:46.19699622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:30:48.164376571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:31:00.385276725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:45902"
ts=2026-05-13T13:31:16.19792082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:31:43.436644777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34530"
ts=2026-05-13T13:31:46.199539409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:31:47.026987111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:31:47.028568549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:32:00.38772644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:47034"
ts=2026-05-13T13:32:16.200929595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:32:23.535059535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:32:44.087733302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49018"
ts=2026-05-13T13:32:46.202543348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:33:15.509159783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:33:16.204354558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:33:43.440310173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37844"
ts=2026-05-13T13:33:46.205190181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:33:51.870049181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:34:13.44142712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38516"
ts=2026-05-13T13:34:16.206746033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=error ts=2026-05-13T13:34:17.478452182Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:34:17.48044458Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:34:17.482225038Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:34:17.482245715Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:34:30.394319518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:51810"
level=debug ts=2026-05-13T13:34:30.465621157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:34:43.443112619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35500"
ts=2026-05-13T13:34:44.09361421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43724"
ts=2026-05-13T13:34:46.20780477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:35:00.395220028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57170"
ts=2026-05-13T13:35:13.445249594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46480"
ts=2026-05-13T13:35:16.209714374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:35:27.740689932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:35:27.742314188Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:35:46.210613071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:36:16.211629716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:36:18.840693421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:36:46.212429886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:37:00.874593222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:37:16.214084194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:37:46.215276054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:37:47.845826897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:38:00.403263041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56730"
ts=2026-05-13T13:38:16.216386577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:38:24.412285903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:38:43.454456592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44066"
ts=2026-05-13T13:38:44.105036239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48812"
ts=2026-05-13T13:38:46.217624313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:39:08.28541626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:39:08.28700582Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:39:14.106462975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47330"
ts=2026-05-13T13:39:16.218916418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:39:30.407458232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48136"
ts=2026-05-13T13:39:44.107619416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49280"
ts=2026-05-13T13:39:46.220554339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:40:08.564937345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:40:14.109387137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40448"
ts=2026-05-13T13:40:16.222102701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:40:46.223785043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:40:53.205215254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:41:00.410902906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54196"
ts=2026-05-13T13:41:16.225220707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:41:44.382826218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:41:46.227123226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:42:13.464559598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40966"
ts=2026-05-13T13:42:16.228202532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:42:30.945945029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:42:43.46611798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41116"
ts=2026-05-13T13:42:46.230127063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:43:13.467428433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34158"
ts=2026-05-13T13:43:16.23130765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:43:17.186477312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:43:17.188159223Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:43:43.468986579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60030"
ts=2026-05-13T13:43:46.232786639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:44:13.470141926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:43394"
ts=2026-05-13T13:44:14.120687783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51884"
ts=2026-05-13T13:44:16.234210194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=error ts=2026-05-13T13:44:17.476393531Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:44:17.477914675Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:44:17.479589682Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:44:17.479612227Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:44:17.533562288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:44:43.471945838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59506"
ts=2026-05-13T13:44:44.122303224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41356"
ts=2026-05-13T13:44:46.235564642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:45:00.422712756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:40940"
level=debug ts=2026-05-13T13:45:03.923790216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:45:16.236763123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:45:30.424214335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56468"
ts=2026-05-13T13:45:44.124688631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47006"
level=debug ts=2026-05-13T13:45:45.816043386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:45:46.23848553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:46:13.476001328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55644"
ts=2026-05-13T13:46:16.239708449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:46:27.392148469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:46:44.126933556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35914"
ts=2026-05-13T13:46:46.240712274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:47:12.916215964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:47:12.917806555Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:47:16.241738608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:47:30.429641325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36628"
ts=2026-05-13T13:47:43.479367849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50488"
ts=2026-05-13T13:47:44.129228916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:54190"
ts=2026-05-13T13:47:46.242562655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:48:14.772123814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:48:16.244110779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:48:44.131958969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:36632"
ts=2026-05-13T13:48:46.245334962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:49:09.850470666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:49:14.133378547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55364"
ts=2026-05-13T13:49:16.246565944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:49:43.484899251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40992"
ts=2026-05-13T13:49:44.135460843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56650"
ts=2026-05-13T13:49:46.247511797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:50:00.436489915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:50656"
level=debug ts=2026-05-13T13:50:08.073574216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:50:16.248744044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:50:44.139024219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39888"
ts=2026-05-13T13:50:46.250391649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:50:56.042822913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:51:13.488627957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56204"
ts=2026-05-13T13:51:14.14024372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46202"
ts=2026-05-13T13:51:16.251361987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:51:30.439817856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36368"
ts=2026-05-13T13:51:43.49041428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45642"
ts=2026-05-13T13:51:44.142042043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49498"
ts=2026-05-13T13:51:46.252541675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:51:47.692561042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:51:47.694158905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:52:14.143544908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48254"
ts=2026-05-13T13:52:16.253685351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:52:46.255400624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:52:48.178589191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:53:16.256773673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:53:34.589997561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:53:46.258232177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:54:14.149375869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47094"
ts=2026-05-13T13:54:16.259777276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=error ts=2026-05-13T13:54:17.477246749Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:54:17.479136525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:54:17.480925478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:54:17.480945826Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:54:35.013844282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:54:46.261542376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:55:16.262770987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:55:24.300010711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:55:43.500996726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:54830"
ts=2026-05-13T13:55:46.263766473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:56:10.739616162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:56:10.741258809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:56:16.265142929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:56:30.452214878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36720"
ts=2026-05-13T13:56:44.155308444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41916"
ts=2026-05-13T13:56:46.26616059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:56:50.489004634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:57:13.505951224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58460"
ts=2026-05-13T13:57:16.267607544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:57:32.921698658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:57:44.157946551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58030"
ts=2026-05-13T13:57:46.26910698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:58:00.455978498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:38226"
ts=2026-05-13T13:58:13.508823693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42556"
ts=2026-05-13T13:58:14.159656876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:37138"
ts=2026-05-13T13:58:16.269980046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T13:58:35.004651593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:58:46.271346892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T13:59:00.458043163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:51186"
level=debug ts=2026-05-13T13:59:14.861091581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:59:16.272886514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:59:43.513284185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50212"
ts=2026-05-13T13:59:46.274351064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:59:54.005020438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:59:54.006525648Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:00:16.276050167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:00:30.461904391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54922"
ts=2026-05-13T14:00:46.277164526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:00:50.129434452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:01:00.463624483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48452"
ts=2026-05-13T14:01:13.517108854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35636"
ts=2026-05-13T14:01:16.278837301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:01:35.972096862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:01:46.279647415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:02:14.172148963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59212"
ts=2026-05-13T14:02:16.280794341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:02:32.765293594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:02:43.521027952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53694"
ts=2026-05-13T14:02:46.281830491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:03:16.283113815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:03:21.449985965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:03:46.284662556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:03:57.211948617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:03:57.213797359Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:04:14.176946158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49008"
ts=2026-05-13T14:04:16.285967011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=error ts=2026-05-13T14:04:17.477320686Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:04:17.478968726Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:04:17.480495318Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:04:17.480518937Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:04:30.47160047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:51076"
level=debug ts=2026-05-13T14:04:44.10612865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:04:46.287833217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:05:13.528296526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38480"
ts=2026-05-13T14:05:14.178942479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41146"
ts=2026-05-13T14:05:16.28914797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:05:20.713016541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:05:46.290671418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:06:05.339164899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:06:14.18173597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55154"
ts=2026-05-13T14:06:16.292117869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:06:30.476513838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52532"
ts=2026-05-13T14:06:43.531668521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33100"
level=debug ts=2026-05-13T14:06:45.521855309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:06:46.293149999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:07:00.477556231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56186"
ts=2026-05-13T14:07:14.184298748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:46696"
ts=2026-05-13T14:07:16.294191832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:07:35.819840945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:07:35.821382165Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:07:43.534166353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52982"
ts=2026-05-13T14:07:44.18580085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56552"
ts=2026-05-13T14:07:46.295810777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:08:14.187273179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47074"
ts=2026-05-13T14:08:16.297363583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:08:25.596608986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:08:46.298382724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:09:08.139082687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:09:16.300111992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:09:46.300818222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:09:52.372075215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:10:00.485366085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39624"
ts=2026-05-13T14:10:14.192148613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52980"
ts=2026-05-13T14:10:16.302356216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:10:28.093077143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:10:30.486891502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33376"
ts=2026-05-13T14:10:44.193517724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43022"
ts=2026-05-13T14:10:46.303278534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:11:03.579774737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:11:03.581281109Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:11:13.542790472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55494"
ts=2026-05-13T14:11:16.304894799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:11:39.190679963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:11:46.306733572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:12:16.308167153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:12:29.990592568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:12:46.309146312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:13:00.493057851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:34326"
ts=2026-05-13T14:13:13.54830253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41684"
ts=2026-05-13T14:13:16.310823177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:13:29.702874931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:13:30.494367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:37636"
ts=2026-05-13T14:13:43.549339274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36740"
ts=2026-05-13T14:13:44.201967858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53620"
ts=2026-05-13T14:13:46.312334282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:14:00.496072774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36816"
ts=2026-05-13T14:14:13.550853231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47584"
ts=2026-05-13T14:14:16.313519856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:14:17.30124863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:14:17.475964715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:14:17.47758449Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:14:17.479148204Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:14:17.479170102Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:14:43.552094302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60432"
ts=2026-05-13T14:14:46.314999196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:15:00.498831847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54132"
level=debug ts=2026-05-13T14:15:01.742945346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:15:01.744590134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:15:16.316109379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:15:30.500474759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:35252"
ts=2026-05-13T14:15:43.555395945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52038"
ts=2026-05-13T14:15:44.206956113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56270"
ts=2026-05-13T14:15:46.31795212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:16:00.502024338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57600"
level=debug ts=2026-05-13T14:16:02.148999973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:16:13.556734501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:59276"
ts=2026-05-13T14:16:16.319481717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:16:46.321256896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:16:55.840811029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:17:16.322719147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:17:44.212384697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:44234"
ts=2026-05-13T14:17:46.32454085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:17:51.157244972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:18:13.562236095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60944"
ts=2026-05-13T14:18:16.32611665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:18:43.564023284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58494"
level=debug ts=2026-05-13T14:18:43.8360607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:18:46.327662388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:19:13.56544736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51938"
ts=2026-05-13T14:19:14.21666748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59774"
ts=2026-05-13T14:19:16.328788408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:19:45.119029862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:19:45.120985992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:19:46.330373716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:20:00.512444107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53712"
ts=2026-05-13T14:20:16.331427703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:20:30.513513643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48162"
ts=2026-05-13T14:20:43.569577468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41670"
ts=2026-05-13T14:20:44.220343599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:50782"
level=debug ts=2026-05-13T14:20:44.84363732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:20:46.332465239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:21:13.570838435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57166"
ts=2026-05-13T14:21:14.22133861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:57416"
ts=2026-05-13T14:21:16.333607611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
ts=2026-05-13T14:21:30.515870196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53398"
level=debug ts=2026-05-13T14:21:30.526320615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:21:43.572490743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38934"
ts=2026-05-13T14:21:46.335123812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:22:16.336732685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15e5a2c0 10.128.0.58:7946"
level=debug ts=2026-05-13T14:22:21.353413937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:22:43.575989458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40874"
ts=2026-05-13T14:22:44.225273109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59074"
ts=2026-05-13T14:22:46.338476143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:23:13.576936921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46060"
ts=2026-05-13T14:23:14.226608485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59028"
level=debug ts=2026-05-13T14:23:14.591782815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"