level=info ts=2026-04-24T22:36:01.761700536Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-24T22:36:01.761745113Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-24T22:36:01.763032917Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-24T22:36:01.763497039Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T22:36:01.763610557Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
level=info ts=2026-04-24T22:36:01.76517583Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-adc48d01
ts=2026-04-24T22:36:01.765247616Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-24T22:36:01.765262699Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=debug ts=2026-04-24T22:36:01.765279598Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=info ts=2026-04-24T22:36:01.765440775Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-24T22:36:01.765040032Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T22:36:01.765722396Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
ts=2026-04-24T22:36:01.775136357Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T22:36:01.775167683Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-24T22:36:01.775192629Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.75416ms
level=error ts=2026-04-24T22:36:01.775203787Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-24T22:36:01.775215069Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-24T22:36:01.779529693Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:36:01.779551903Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-24T22:36:01.781485311Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-24T22:36:01.78152124Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-24T22:36:01.781539782Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-24T22:36:01.781552777Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-24T22:36:01.781643547Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-24T22:36:01.781687948Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-24T22:36:01.781782408Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-24T22:36:01.781792763Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-24T22:36:01.78192001Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-24T22:36:01.781932043Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-24T22:36:01.781966061Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-24T22:36:01.782004191Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-24T22:36:01.782027648Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-24T22:36:01.782209723Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-24T22:36:01.782220224Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-24T22:36:01.782236504Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-24T22:36:02.782656448Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-24T22:36:02.982763743Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-24T22:36:02.982835893Z caller=loki.go:599 msg="Loki started" startup_time=1.313837493s
ts=2026-04-24T22:36:03.658379896Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:36:03.658405627Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:36:05.732281208Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:36:05.732312997Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-24T22:36:07.983377472Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-24T22:36:07.983464231Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-24T22:36:09.990097189Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:36:09.990126536Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-24T22:36:18.288867603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:36:25.262658565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.48:7946"
ts=2026-04-24T22:36:25.263642473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.74:7946"
ts=2026-04-24T22:36:25.263711925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38356"
ts=2026-04-24T22:36:25.264801615Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.73:7946: dial tcp 10.129.0.73:7946: connect: connection refused"
ts=2026-04-24T22:36:25.265655893Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.72:7946: dial tcp 10.129.0.72:7946: connect: connection refused"
ts=2026-04-24T22:36:25.266242222Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused"
ts=2026-04-24T22:36:25.267000399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.68:7946"
level=info ts=2026-04-24T22:36:25.26743533Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.49219631s
ts=2026-04-24T22:36:26.357339574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:45562"
ts=2026-04-24T22:36:26.408066462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46986"
level=debug ts=2026-04-24T22:36:42.691885196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:36:54.152190309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T22:37:05.372667442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:37:24.154268709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:37:27.643183541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34582"
level=debug ts=2026-04-24T22:37:28.793355481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:37:35.306099522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39632"
ts=2026-04-24T22:37:54.156039066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T22:38:00.889188493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T22:38:00.891799523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:38:24.157629225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:38:29.620546565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33762"
level=debug ts=2026-04-24T22:38:48.901255479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:38:54.159563685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:38:57.649340005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:42420"
ts=2026-04-24T22:38:59.621602153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55972"
ts=2026-04-24T22:39:05.312216536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34264"
ts=2026-04-24T22:39:24.161522194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:39:29.622669347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:32892"
ts=2026-04-24T22:39:54.163289813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T22:39:58.427962506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:40:24.165311393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:40:27.655650092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:60780"
ts=2026-04-24T22:40:54.167138095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:40:57.658236025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:44206"
level=debug ts=2026-04-24T22:41:08.178026353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:41:24.16943694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:41:54.171380545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:41:59.630190399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:45582"
level=debug ts=2026-04-24T22:41:59.925856824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:42:24.1732541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:42:54.175142542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T22:42:58.320840482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T22:42:58.323461272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:43:24.17622457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:43:35.329050815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60574"
level=debug ts=2026-04-24T22:43:42.887255981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:43:54.177586884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:44:18.26415695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:44:24.178693813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:44:27.672851021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:49704"
ts=2026-04-24T22:44:29.639772556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50688"
ts=2026-04-24T22:44:35.332312924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49598"
ts=2026-04-24T22:44:54.180237718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:44:59.64112867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50060"
level=debug ts=2026-04-24T22:45:07.280405596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:45:24.181341814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:45:27.67744713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:44988"
ts=2026-04-24T22:45:29.642517275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54560"
level=debug ts=2026-04-24T22:45:50.686559063Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:45:54.182763259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=info ts=2026-04-24T22:46:07.984401794Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-24T22:46:07.988130552Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:46:07.990374487Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:46:07.992296119Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:46:07.992321206Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T22:46:07.992332666Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-24T22:46:24.184156284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:46:27.681729004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:54108"
ts=2026-04-24T22:46:29.645950161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38594"
ts=2026-04-24T22:46:35.338780454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48772"
level=debug ts=2026-04-24T22:46:43.775820427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:46:43.777646787Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:46:54.186065423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:47:24.187455288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:47:27.685998131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:51350"
ts=2026-04-24T22:47:35.34298028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40016"
level=debug ts=2026-04-24T22:47:43.476103501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:47:54.189274155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:48:24.191013382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:48:27.689923798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:47142"
ts=2026-04-24T22:48:35.346341605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38958"
level=debug ts=2026-04-24T22:48:39.565068741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:48:54.193465257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:48:57.692560677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52190"
ts=2026-04-24T22:49:05.34719279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57848"
ts=2026-04-24T22:49:24.195288245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:49:31.267408363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:49:35.348251738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36910"
ts=2026-04-24T22:49:54.19699601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T22:50:19.867944954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:50:24.198663614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:50:35.352053255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43360"
ts=2026-04-24T22:50:54.200242631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:50:59.663048156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39918"
ts=2026-04-24T22:51:05.353756094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35490"
level=debug ts=2026-04-24T22:51:16.522453199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:51:16.524025688Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:51:24.201664866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:51:35.35485373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42132"
ts=2026-04-24T22:51:54.203387662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:51:59.666727243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39518"
level=debug ts=2026-04-24T22:52:00.018905591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:52:05.35585001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52524"
ts=2026-04-24T22:52:24.205241997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:52:35.357054874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39274"
ts=2026-04-24T22:52:54.20693194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:52:59.670221614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:44528"
level=debug ts=2026-04-24T22:52:59.859568564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:53:24.208693156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:53:29.671601418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46268"
ts=2026-04-24T22:53:54.210695937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:53:59.672759422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55722"
level=debug ts=2026-04-24T22:54:02.024744392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:54:24.212949685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:54:39.342504724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:54:54.214805797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:55:05.36591246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40220"
ts=2026-04-24T22:55:24.216687513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:55:26.84558168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:55:26.847272472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:55:54.217689664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:55:57.721603272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:37020"
ts=2026-04-24T22:55:59.679845677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47308"
ts=2026-04-24T22:56:05.368492134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36654"
level=error ts=2026-04-24T22:56:07.997031344Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:56:07.999032988Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:56:08.001285Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:56:08.001316804Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:56:24.218709219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:56:28.282295466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:56:54.220540151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T22:56:57.727377221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:36244"
ts=2026-04-24T22:57:24.222641762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T22:57:26.282131468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:57:54.224370373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:57:59.68548736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59238"
level=debug ts=2026-04-24T22:58:03.446947732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:58:24.225361593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T22:58:35.375578701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:32960"
ts=2026-04-24T22:58:54.226346437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T22:58:57.283142151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:59:05.376707011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44628"
ts=2026-04-24T22:59:24.227454425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T22:59:47.581265355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:59:47.583394085Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:59:54.229061478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T22:59:57.739609537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:58156"
ts=2026-04-24T22:59:59.691726368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38234"
ts=2026-04-24T23:00:24.231194391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:00:27.741600872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:57868"
ts=2026-04-24T23:00:29.693719876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55892"
level=debug ts=2026-04-24T23:00:44.527038537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:00:54.232809295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:00:57.744900169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:41876"
level=debug ts=2026-04-24T23:01:19.926966143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:01:24.235478487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:01:27.748808634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:60040"
ts=2026-04-24T23:01:54.23767056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:01:57.75143999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52854"
level=debug ts=2026-04-24T23:01:59.264670462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:02:24.239530097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:02:27.753922004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:46594"
ts=2026-04-24T23:02:54.241203771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:02:59.703266507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37968"
level=debug ts=2026-04-24T23:02:59.706490098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:03:24.242454528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
level=debug ts=2026-04-24T23:03:34.267704045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:03:34.269521613Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:03:35.39387706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40442"
ts=2026-04-24T23:03:54.243660332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:04:05.395318871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56232"
ts=2026-04-24T23:04:24.244964355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:04:27.762345047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:39144"
level=debug ts=2026-04-24T23:04:28.525208314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:04:54.246160231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:04:57.764780904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52792"
level=debug ts=2026-04-24T23:05:15.900789339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:05:24.247362855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:05:35.403093508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41282"
ts=2026-04-24T23:05:54.248174608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:05:59.712611242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40044"
ts=2026-04-24T23:06:05.406870337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36344"
level=error ts=2026-04-24T23:06:08.008069552Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:06:08.01239599Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:06:08.016056231Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:06:08.016110573Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-24T23:06:09.538723282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:06:24.249882847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:06:54.251660262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:07:05.410004707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47752"
level=debug ts=2026-04-24T23:07:10.20427065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:07:24.253143276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:07:29.718648103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57152"
ts=2026-04-24T23:07:54.25519427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:08:03.794751852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:08:03.797058535Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:08:24.257122808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:08:35.414378225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34992"
ts=2026-04-24T23:08:54.258883219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:08:57.782609006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45790"
ts=2026-04-24T23:08:59.722812307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:52616"
level=debug ts=2026-04-24T23:09:03.293222049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:09:05.415961638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57938"
ts=2026-04-24T23:09:24.260249199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:09:54.262627952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:09:57.78706288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52574"
level=debug ts=2026-04-24T23:10:01.390736974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:10:05.419136524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52634"
ts=2026-04-24T23:10:24.264583396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:10:43.726983513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:10:54.266333079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:10:59.729414093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:48996"
ts=2026-04-24T23:11:05.42308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52060"
ts=2026-04-24T23:11:24.26837173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:11:44.04803052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:11:54.27011001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d865ce86 10.129.0.75:7946"
ts=2026-04-24T23:12:05.427117982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54576"
ts=2026-04-24T23:12:24.272007791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:12:27.797727705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:56488"
level=debug ts=2026-04-24T23:12:32.371475944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:12:32.373685266Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:12:51.103977915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58476"
ts=2026-04-24T23:12:51.111561226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58478"
ts=2026-04-24T23:12:54.27618168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:12:58.76681691Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d865ce86 (timeout reached)"
ts=2026-04-24T23:12:59.735968505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41570"
ts=2026-04-24T23:13:01.766785398Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d865ce86 has failed, no acks received"
ts=2026-04-24T23:13:08.766500273Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d865ce86 (timeout reached)"
ts=2026-04-24T23:13:11.766972202Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d865ce86 has failed, no acks received"
level=debug ts=2026-04-24T23:13:13.64590444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:13:21.76688713Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-d865ce86 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:13:24.27801849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:13:54.279921882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:13:57.80487051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55084"
ts=2026-04-24T23:13:59.739283609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:57222"
level=debug ts=2026-04-24T23:14:05.539729011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:14:24.281803894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:14:46.731546112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:14:54.283544613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:15:01.046183838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60670"
ts=2026-04-24T23:15:24.284995858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:15:27.811117838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52808"
level=debug ts=2026-04-24T23:15:35.860135967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:15:54.28604463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-166bb25c 10.129.0.75:7946"
ts=2026-04-24T23:15:59.748248269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55690"
ts=2026-04-24T23:16:01.049915073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46044"
level=error ts=2026-04-24T23:16:07.997481978Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:16:07.999350501Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:16:08.00109346Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:16:08.001121306Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:16:24.287775589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-166bb25c 10.129.0.75:7946"
ts=2026-04-24T23:16:27.815339099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:37284"
ts=2026-04-24T23:16:31.051415781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46540"
level=debug ts=2026-04-24T23:16:37.613946253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:16:37.616194381Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:16:54.28936441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:17:24.294687544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:17:36.388733017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:17:54.298961816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-166bb25c 10.129.0.75:7946"
ts=2026-04-24T23:17:59.755217688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60492"
level=debug ts=2026-04-24T23:18:15.882130016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:18:24.300818904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:18:31.057485067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59134"
ts=2026-04-24T23:18:54.302577957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:18:59.758749415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49936"
ts=2026-04-24T23:19:01.058471806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44440"
level=debug ts=2026-04-24T23:19:01.062619104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:19:24.30481812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-166bb25c 10.129.0.75:7946"
ts=2026-04-24T23:19:29.759699885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60254"
level=debug ts=2026-04-24T23:19:47.988281639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:19:54.307139817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:20:01.061528631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51792"
ts=2026-04-24T23:20:24.308913992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:20:30.530523988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:20:30.532704687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:20:54.309906775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-166bb25c 10.129.0.75:7946"
ts=2026-04-24T23:20:57.835842853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:57920"
level=debug ts=2026-04-24T23:21:13.930202537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:21:24.311636417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:21:27.837917725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:33912"
ts=2026-04-24T23:21:29.76618247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:40598"
ts=2026-04-24T23:21:38.766481683Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-166bb25c (timeout reached)"
ts=2026-04-24T23:21:41.765896989Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-166bb25c has failed, no acks received"
ts=2026-04-24T23:21:51.029715357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34592"
ts=2026-04-24T23:21:51.037040198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34594"
ts=2026-04-24T23:21:54.313053817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:22:01.766478464Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-166bb25c as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-24T23:22:12.985700733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:22:24.314733756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0af30281 10.129.0.75:7946"
ts=2026-04-24T23:22:27.843768306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:47642"
ts=2026-04-24T23:22:29.770174216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36008"
ts=2026-04-24T23:22:54.316354514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:23:06.986056614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:23:24.317726178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:23:29.773341137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:55302"
level=debug ts=2026-04-24T23:23:53.752943898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:23:54.31961206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0af30281 10.129.0.75:7946"
ts=2026-04-24T23:23:57.850811421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:35822"
ts=2026-04-24T23:24:24.321294894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:24:29.776875727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51690"
level=debug ts=2026-04-24T23:24:33.66789137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:24:33.669799858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:24:36.074690375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56280"
ts=2026-04-24T23:24:54.323072721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:24:57.855291789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:59348"
ts=2026-04-24T23:25:24.324657068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:25:29.779861384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37998"
level=debug ts=2026-04-24T23:25:30.5043424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:25:54.326050478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=error ts=2026-04-24T23:26:07.997544977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:26:08.000098707Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:26:08.002625986Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:26:08.002671795Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-24T23:26:12.226004624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:26:24.327538339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0af30281 10.129.0.75:7946"
ts=2026-04-24T23:26:27.861388677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:36216"
ts=2026-04-24T23:26:54.32858409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0af30281 10.129.0.75:7946"
level=debug ts=2026-04-24T23:27:07.063625505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:27:24.329830301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:27:42.720226486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:27:53.766470143Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0af30281 (timeout reached)"
ts=2026-04-24T23:27:54.331668954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:27:56.766852004Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0af30281 has failed, no acks received"
ts=2026-04-24T23:27:59.011851986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46170"
ts=2026-04-24T23:27:59.0195971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46182"
ts=2026-04-24T23:28:03.766785937Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0af30281 (timeout reached)"
ts=2026-04-24T23:28:06.766606888Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0af30281 has failed, no acks received"
level=debug ts=2026-04-24T23:28:17.817317565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:28:17.819303338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:28:24.333334405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:28:29.792157619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42162"
ts=2026-04-24T23:28:31.986201743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60042"
ts=2026-04-24T23:28:54.334878745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:29:10.923395294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:29:24.336534719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:29:27.874805891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:54776"
ts=2026-04-24T23:29:54.338773811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:29:57.87701382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:42120"
ts=2026-04-24T23:29:59.798477542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:37114"
level=debug ts=2026-04-24T23:30:05.495186356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:30:24.340740865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:30:54.342798008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:30:59.804278703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51682"
ts=2026-04-24T23:31:01.995297035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60802"
level=debug ts=2026-04-24T23:31:02.989381471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:31:24.344763479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:31:29.805958531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:51910"
ts=2026-04-24T23:31:31.996409016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47410"
ts=2026-04-24T23:31:54.346511885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:32:02.195738633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:32:24.348029048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b50bb7e7 10.129.0.75:7946"
ts=2026-04-24T23:32:54.349301543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:32:57.889559505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:36178"
level=debug ts=2026-04-24T23:33:03.504534821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:33:03.506247602Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:33:24.350150694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b50bb7e7 10.129.0.75:7946"
ts=2026-04-24T23:33:27.8920135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:44454"
ts=2026-04-24T23:33:29.813566408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:60028"
ts=2026-04-24T23:33:32.003355467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48760"
ts=2026-04-24T23:33:54.352498588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:33:55.505072645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:33:57.894070283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:49420"
ts=2026-04-24T23:33:59.814981823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56842"
ts=2026-04-24T23:34:24.359727417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:34:39.040715877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:34:54.361969009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:35:17.936162639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:35:24.363458266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b50bb7e7 10.129.0.75:7946"
ts=2026-04-24T23:35:27.901354138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:59784"
ts=2026-04-24T23:35:54.364832442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=error ts=2026-04-24T23:36:07.996721556Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:36:07.999291709Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:36:08.001512244Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:36:08.001546384Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-24T23:36:18.144747842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:36:24.366517923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b50bb7e7 10.129.0.75:7946"
ts=2026-04-24T23:36:27.905528918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:37306"
ts=2026-04-24T23:36:54.367720232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:37:16.756865106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:37:16.758375218Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:37:23.766082089Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b50bb7e7 (timeout reached)"
ts=2026-04-24T23:37:24.368880075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:37:26.766971057Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b50bb7e7 has failed, no acks received"
ts=2026-04-24T23:37:29.827083038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:32900"
ts=2026-04-24T23:37:33.767060265Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b50bb7e7 (timeout reached)"
ts=2026-04-24T23:37:36.76701761Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b50bb7e7 has failed, no acks received"
ts=2026-04-24T23:37:39.172013613Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b50bb7e7 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:37:54.370702026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:37:57.001197377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38986"
ts=2026-04-24T23:37:57.009011023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38996"
level=debug ts=2026-04-24T23:38:00.834539628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:38:24.372542641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:38:27.912987629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50732"
ts=2026-04-24T23:38:29.831178932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47502"
ts=2026-04-24T23:38:38.477882717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57788"
ts=2026-04-24T23:38:54.373851901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:38:55.937166731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:39:24.375026734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:39:35.669937641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:39:54.376406166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-749ca795 10.129.0.75:7946"
ts=2026-04-24T23:40:24.378014631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:40:27.921446802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:58070"
ts=2026-04-24T23:40:29.838498767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:56800"
level=debug ts=2026-04-24T23:40:32.758250145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:40:54.379196599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:40:59.839548237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59034"
ts=2026-04-24T23:41:08.490684426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44080"
ts=2026-04-24T23:41:24.381128918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:41:27.539461024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:41:27.541280972Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:41:27.925910088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:41886"
ts=2026-04-24T23:41:54.38241545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:41:57.927929478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:52158"
level=debug ts=2026-04-24T23:42:18.006321638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:42:24.383373854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:42:27.930573143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:46004"
ts=2026-04-24T23:42:54.38424457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:43:18.073721906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:43:24.38633422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:43:27.934574581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:58152"
ts=2026-04-24T23:43:38.499127491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59222"
ts=2026-04-24T23:43:54.39241644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:44:06.233304397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:44:24.39388962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:44:29.85150588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39376"
ts=2026-04-24T23:44:38.502330188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38164"
ts=2026-04-24T23:44:54.396210469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:44:59.648313134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:45:24.39770978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-749ca795 10.129.0.75:7946"
ts=2026-04-24T23:45:54.399175868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:45:56.729926692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:45:56.731510426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:46:07.996149699Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:46:07.998447184Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:46:08.000389689Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:46:08.000417007Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:46:24.400356982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-749ca795 10.129.0.75:7946"
ts=2026-04-24T23:46:27.948755481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50338"
ts=2026-04-24T23:46:29.85683761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:33320"
level=debug ts=2026-04-24T23:46:35.346706558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:46:54.401413419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-24T23:47:22.270021025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:47:24.403465591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:47:29.859303122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:45534"
ts=2026-04-24T23:47:54.40485447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-749ca795 10.129.0.75:7946"
ts=2026-04-24T23:47:57.954010707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:56066"
level=debug ts=2026-04-24T23:47:58.055576593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:48:02.105000005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39804"
ts=2026-04-24T23:48:02.111729582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39812"
ts=2026-04-24T23:48:03.767041906Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-749ca795 (timeout reached)"
ts=2026-04-24T23:48:06.766800837Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-749ca795 has failed, no acks received"
ts=2026-04-24T23:48:13.767018183Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-749ca795 (timeout reached)"
ts=2026-04-24T23:48:16.767021374Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-749ca795 has failed, no acks received"
ts=2026-04-24T23:48:24.406868829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:48:26.767605363Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-749ca795 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:48:28.766410364Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-749ca795 (timeout reached)"
ts=2026-04-24T23:48:31.766704833Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-749ca795 has failed, no acks received"
level=debug ts=2026-04-24T23:48:37.201320802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:48:54.408930665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:49:15.796706555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:49:15.798553738Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:49:21.686364772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39150"
ts=2026-04-24T23:49:24.411525279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:49:51.687371716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43808"
ts=2026-04-24T23:49:54.413815607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:49:59.868300427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59444"
level=debug ts=2026-04-24T23:50:10.594550311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:50:21.689006669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60466"
ts=2026-04-24T23:50:24.416159113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:50:27.96339745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55470"
ts=2026-04-24T23:50:29.869867341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:45046"
ts=2026-04-24T23:50:54.417904662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:50:56.501579889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:50:57.965652311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:35906"
ts=2026-04-24T23:51:24.419505417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:51:27.967483829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45670"
ts=2026-04-24T23:51:54.421745198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:51:57.252778537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:51:57.969300474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:37644"
ts=2026-04-24T23:51:59.874935298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47780"
ts=2026-04-24T23:52:21.695937849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48818"
ts=2026-04-24T23:52:24.423877467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
level=debug ts=2026-04-24T23:52:38.029579934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:52:51.697536495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40632"
ts=2026-04-24T23:52:54.425786162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:52:57.973326061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:59464"
level=debug ts=2026-04-24T23:53:14.84572701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:53:14.847300229Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:53:24.427935585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:53:27.975928485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:47264"
ts=2026-04-24T23:53:54.429407806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-24T23:53:57.978017825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50022"
level=debug ts=2026-04-24T23:54:11.892995342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:54:24.430381193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-24T23:54:29.88446894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47920"
ts=2026-04-24T23:54:54.431688448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-24T23:54:57.982529701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:36932"
ts=2026-04-24T23:54:59.885686606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59206"
level=debug ts=2026-04-24T23:55:05.835686745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:55:24.433823236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-24T23:55:54.435280076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:55:57.786985359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:56:07.997274086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:56:07.999561347Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:56:08.001644729Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:56:08.001680814Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:56:21.710171429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38654"
ts=2026-04-24T23:56:24.436827144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-24T23:56:29.890459047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42580"
level=debug ts=2026-04-24T23:56:35.266535445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:56:51.711896553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59544"
ts=2026-04-24T23:56:54.438491805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:57:19.566854795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:57:19.568855334Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:57:24.439863949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-24T23:57:27.993368091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:46576"
ts=2026-04-24T23:57:29.893184116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58852"
ts=2026-04-24T23:57:54.441328114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:57:56.580323331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:58:24.443830732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-24T23:58:40.53373997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:58:51.718760317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:32778"
ts=2026-04-24T23:58:54.445870119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:59:21.720187847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38632"
level=debug ts=2026-04-24T23:59:23.47710197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:59:24.44697311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-24T23:59:28.00076539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:39240"
ts=2026-04-24T23:59:51.721704243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57010"
ts=2026-04-24T23:59:54.448312574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-25T00:00:21.599694224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:00:24.45005532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:00:54.451979441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d93902 10.129.0.75:7946"
ts=2026-04-25T00:00:57.586913661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43326"
ts=2026-04-25T00:00:57.596289888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43334"
ts=2026-04-25T00:00:58.765815373Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-74d93902 (timeout reached)"
ts=2026-04-25T00:01:01.765857995Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-74d93902 has failed, no acks received"
ts=2026-04-25T00:01:08.766003902Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-74d93902 (timeout reached)"
ts=2026-04-25T00:01:11.766202665Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-74d93902 has failed, no acks received"
level=debug ts=2026-04-25T00:01:14.879914598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:01:14.882460121Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:01:21.765945964Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-74d93902 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-25T00:01:24.453237081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:01:54.45515443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:01:59.909608705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54592"
level=debug ts=2026-04-25T00:02:08.154574118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:02:22.043109194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42314"
ts=2026-04-25T00:02:24.457236072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:02:28.013709747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:58354"
ts=2026-04-25T00:02:29.910741166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36206"
level=debug ts=2026-04-25T00:02:47.624654557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:02:54.459108298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:02:58.016228494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45856"
ts=2026-04-25T00:02:59.912495571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:47588"
ts=2026-04-25T00:03:22.047046447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36384"
level=debug ts=2026-04-25T00:03:22.995726642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:03:24.460747377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:03:54.462237382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:04:24.463335316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-25T00:04:24.955462974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:04:54.465229495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:04:59.919691585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42522"
level=debug ts=2026-04-25T00:05:11.312920406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:05:11.314534433Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:05:22.053003877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43356"
ts=2026-04-25T00:05:24.466833226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:05:29.921518617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:49776"
ts=2026-04-25T00:05:54.468373283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-25T00:05:57.623724586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:05:58.029276746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:47230"
ts=2026-04-25T00:05:59.923165137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36976"
level=error ts=2026-04-25T00:06:07.996577213Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:06:07.99841648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:06:08.000316532Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:06:08.000345118Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:06:24.470490923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-25T00:06:44.558799113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:06:54.471828774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:06:58.033040974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:46094"
ts=2026-04-25T00:07:24.473696204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-25T00:07:28.842333975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:07:29.927488069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:42314"
ts=2026-04-25T00:07:52.061706497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57242"
ts=2026-04-25T00:07:54.47484306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:07:58.037256034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:59098"
level=debug ts=2026-04-25T00:08:11.135797044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:08:24.476565669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:08:28.03947212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45526"
ts=2026-04-25T00:08:52.064087691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54868"
ts=2026-04-25T00:08:54.477550471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:08:58.042076896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:37392"
level=debug ts=2026-04-25T00:08:59.46664284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:08:59.468316878Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:09:24.479074802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:09:28.043983305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:39238"
ts=2026-04-25T00:09:54.480797734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-25T00:09:57.868448285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:09:58.045636056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34982"
ts=2026-04-25T00:10:22.068898698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35454"
ts=2026-04-25T00:10:24.482496545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49f84d5b 10.129.0.75:7946"
ts=2026-04-25T00:10:28.047185944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50352"
level=debug ts=2026-04-25T00:10:47.162211621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:10:54.48400552Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-49f84d5b failed: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-04-25T00:10:59.93780109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:58066"
ts=2026-04-25T00:11:03.766571902Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-49f84d5b (timeout reached)"
ts=2026-04-25T00:11:06.766852858Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-49f84d5b has failed, no acks received"
ts=2026-04-25T00:11:07.003161548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46730"
ts=2026-04-25T00:11:07.011140089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46734"
ts=2026-04-25T00:11:24.485522908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
level=debug ts=2026-04-25T00:11:44.518834522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:11:54.487433628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:11:59.940449997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:53482"
ts=2026-04-25T00:12:00.036922991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34774"
ts=2026-04-25T00:12:24.488944346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8d775ed8 10.129.0.75:7946"
ts=2026-04-25T00:12:28.054566937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:51400"
level=debug ts=2026-04-25T00:12:30.718325092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:12:54.490771112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:12:58.056556431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:50424"
level=debug ts=2026-04-25T00:13:18.170462822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:13:18.17215029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:13:24.491983146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:13:29.945009863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39410"
ts=2026-04-25T00:13:30.041888417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36394"
ts=2026-04-25T00:13:54.493596588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8d775ed8 10.129.0.75:7946"
ts=2026-04-25T00:13:58.060084659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:44434"
level=debug ts=2026-04-25T00:13:59.49526168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:14:24.494828253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8d775ed8 10.129.0.75:7946"
ts=2026-04-25T00:14:28.062254367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:34414"
ts=2026-04-25T00:14:29.947829502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:50354"
level=debug ts=2026-04-25T00:14:47.408691201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:14:54.497259735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:15:24.499082591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:15:29.951140421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:54542"
level=debug ts=2026-04-25T00:15:30.155090619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:15:54.501218285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:15:58.069123667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:48128"
level=error ts=2026-04-25T00:16:08.00018627Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:16:08.002106138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:16:08.003830715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:16:08.003857915Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:16:24.502579228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-25T00:16:28.552709179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:16:54.504092243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8d775ed8 10.129.0.75:7946"
ts=2026-04-25T00:16:58.073961856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:56974"
ts=2026-04-25T00:17:00.054511898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55332"
level=debug ts=2026-04-25T00:17:13.236946638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:17:13.238891958Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:17:24.505584226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:17:30.056064645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49016"
level=debug ts=2026-04-25T00:17:51.05039823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:17:54.507453432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:17:59.958963824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:46998"
ts=2026-04-25T00:18:00.057236906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51404"
ts=2026-04-25T00:18:24.509392466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:18:30.058821829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57566"
level=debug ts=2026-04-25T00:18:41.403381282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:18:54.510850995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8d775ed8 10.129.0.75:7946"
ts=2026-04-25T00:19:00.060126971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49948"
level=debug ts=2026-04-25T00:19:24.17775429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:19:24.511875453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:19:43.766467923Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8d775ed8 (timeout reached)"
ts=2026-04-25T00:19:46.766701239Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8d775ed8 has failed, no acks received"
ts=2026-04-25T00:19:46.991132112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35870"
ts=2026-04-25T00:19:46.998840128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35876"
ts=2026-04-25T00:19:48.767839714Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8d775ed8 (timeout reached)"
ts=2026-04-25T00:19:51.767555302Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8d775ed8 has failed, no acks received"
ts=2026-04-25T00:19:54.512910705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:19:56.77281666Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8d775ed8 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-25T00:20:05.378675911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:20:24.514557132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05ca9708 10.129.0.75:7946"
ts=2026-04-25T00:20:29.967797637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:48220"
level=debug ts=2026-04-25T00:20:51.828464258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:20:51.830448658Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:20:54.516015919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05ca9708 10.129.0.75:7946"
ts=2026-04-25T00:20:58.092314998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:60700"
ts=2026-04-25T00:20:59.969322996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:36820"
ts=2026-04-25T00:21:24.518203811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:21:28.094563628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:45062"
ts=2026-04-25T00:21:41.22616694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60576"
level=debug ts=2026-04-25T00:21:50.181486966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:21:54.519904721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-05ca9708 10.129.0.75:7946"
ts=2026-04-25T00:21:58.097047542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55704"
ts=2026-04-25T00:21:59.972713843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:59202"
ts=2026-04-25T00:22:24.521592512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:22:28.099519592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:36652"
ts=2026-04-25T00:22:29.974494362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:39290"
level=debug ts=2026-04-25T00:22:39.071264136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:22:41.228997075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53096"
ts=2026-04-25T00:22:54.522628899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
ts=2026-04-25T00:22:58.101982318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:55690"
ts=2026-04-25T00:22:59.97546389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:38948"
ts=2026-04-25T00:23:24.523984622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hkvvq-00a65c3c 10.129.0.68:7946"
level=debug ts=2026-04-25T00:23:27.767509437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:23:29.977007897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:43592"
ts=2026-04-25T00:23:54.525570802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"
ts=2026-04-25T00:23:59.977958491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.68:41600"
level=debug ts=2026-04-25T00:24:10.616934738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:24:24.527007566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-26pvk-88d27244 10.130.0.48:7946"