level=info ts=2026-04-20T14:28:25.013653795Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T14:28:25.013688199Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T14:28:25.014451076Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T14:28:25.014769Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T14:28:25.014870481Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
level=debug ts=2026-04-20T14:28:25.01621932Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=info ts=2026-04-20T14:28:25.016306866Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-41d5dfa1
level=debug ts=2026-04-20T14:28:25.01630685Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.82 inf=eth0
ts=2026-04-20T14:28:25.016392101Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T14:28:25.016407188Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=debug ts=2026-04-20T14:28:25.016423272Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.82 advertisePort=7946
level=info ts=2026-04-20T14:28:25.01654529Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T14:28:25.024637298Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T14:28:25.024659114Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T14:28:25.024669645Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.125936ms
level=error ts=2026-04-20T14:28:25.024677908Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T14:28:25.024693251Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T14:28:25.028103567Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:28:25.028131039Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T14:28:25.033043311Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T14:28:25.033057888Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T14:28:25.033085508Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-20T14:28:25.033116329Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-20T14:28:25.033121729Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T14:28:25.033162071Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T14:28:25.033307524Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T14:28:25.033336116Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-20T14:28:25.03361071Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T14:28:25.033628125Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T14:28:25.033689089Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T14:28:25.033802576Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T14:28:25.03389365Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T14:28:25.034284059Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T14:28:25.18250723Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T14:28:25.182583523Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T14:28:26.183486938Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T14:28:26.348944454Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T14:28:26.34899632Z caller=loki.go:599 msg="Loki started" startup_time=1.400408524s
ts=2026-04-20T14:28:26.824168286Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:28:26.824207924Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:28:30.402481679Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:28:30.402512362Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T14:28:31.34994516Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T14:28:31.350008538Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T14:28:35.882333291Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T14:28:35.882360307Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T14:28:43.477191501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:28:48.551977392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-04-20T14:28:48.553075062Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-04-20T14:28:48.553822334Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.81:7946: dial tcp 10.128.0.81:7946: connect: connection refused"
ts=2026-04-20T14:28:48.553953308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
ts=2026-04-20T14:28:48.554019609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55178"
ts=2026-04-20T14:28:48.55586623Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.63:7946: dial tcp 10.130.0.63:7946: connect: connection refused"
level=info ts=2026-04-20T14:28:48.555875643Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=23.531176037s
ts=2026-04-20T14:28:50.210734732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33106"
ts=2026-04-20T14:28:51.370939869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45746"
ts=2026-04-20T14:28:53.127918805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49878"
level=debug ts=2026-04-20T14:28:57.71550453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:28:58.128934143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:29:06.720451049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33500"
level=debug ts=2026-04-20T14:29:15.720638665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:29:24.139236881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33370"
ts=2026-04-20T14:29:28.131397672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:29:40.260958522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:29:58.134129276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:30:01.240329981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T14:30:01.24188816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:30:16.346769747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39344"
ts=2026-04-20T14:30:28.136204664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:30:36.72413008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57172"
ts=2026-04-20T14:30:46.347871979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48082"
level=debug ts=2026-04-20T14:30:47.645813407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:30:58.138183832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:31:28.140212907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
ts=2026-04-20T14:31:36.727714575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37016"
level=debug ts=2026-04-20T14:31:37.604978634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:31:46.350157437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43422"
ts=2026-04-20T14:31:54.151438698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60404"
ts=2026-04-20T14:31:58.142401326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:32:16.351694604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44506"
ts=2026-04-20T14:32:28.143721849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:32:36.729738418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35854"
level=debug ts=2026-04-20T14:32:48.950550492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:32:54.156082058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:42816"
ts=2026-04-20T14:32:58.145157617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:33:28.146775959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:33:36.73283649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59272"
ts=2026-04-20T14:33:46.357386939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55128"
level=debug ts=2026-04-20T14:33:55.900092359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T14:33:58.148676453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:34:16.358542064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42314"
ts=2026-04-20T14:34:28.150801667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
ts=2026-04-20T14:34:46.359523841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37984"
level=debug ts=2026-04-20T14:34:51.205146663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T14:34:51.206739532Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:34:54.164546236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46502"
ts=2026-04-20T14:34:58.153043345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:35:16.361175533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55728"
ts=2026-04-20T14:35:28.155231755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:35:36.741065886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37336"
level=debug ts=2026-04-20T14:35:41.309602075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:35:46.362001356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50242"
ts=2026-04-20T14:35:54.16905372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:44522"
ts=2026-04-20T14:35:58.157243508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:36:06.74235627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44452"
ts=2026-04-20T14:36:28.159727388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:36:38.446747131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:36:58.161756766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
ts=2026-04-20T14:37:06.744592616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56196"
ts=2026-04-20T14:37:16.367271527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55580"
ts=2026-04-20T14:37:24.176746305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38514"
level=debug ts=2026-04-20T14:37:26.65172695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:37:28.16268329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:37:36.745539449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39480"
ts=2026-04-20T14:37:58.164513152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
level=debug ts=2026-04-20T14:38:10.879296057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:38:16.370343412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44798"
ts=2026-04-20T14:38:28.16575704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
level=info ts=2026-04-20T14:38:31.351032307Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T14:38:31.355019015Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:38:31.357082503Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:38:31.358847481Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T14:38:31.358889473Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T14:38:31.358903052Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T14:38:36.747922693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34234"
level=debug ts=2026-04-20T14:38:49.776442989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:38:49.778046338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:38:58.167408806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:39:16.3734983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42328"
ts=2026-04-20T14:39:28.169678682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:39:44.624306628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:39:46.375262847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35046"
ts=2026-04-20T14:39:58.171991487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:40:16.376213245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51448"
ts=2026-04-20T14:40:28.1736259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:40:32.867101372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:40:54.194102146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33526"
ts=2026-04-20T14:40:58.175180565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:41:16.3789774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:32794"
level=debug ts=2026-04-20T14:41:23.646714564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:41:24.195979635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:54364"
ts=2026-04-20T14:41:28.176222259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
ts=2026-04-20T14:41:36.759119809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56936"
ts=2026-04-20T14:41:58.178162764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
level=debug ts=2026-04-20T14:42:03.992074768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:42:16.383262738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36768"
ts=2026-04-20T14:42:28.179841819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:42:46.384293722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39784"
level=debug ts=2026-04-20T14:42:48.761297325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T14:42:48.763075726Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T14:42:58.180773798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
ts=2026-04-20T14:43:28.182156711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
level=debug ts=2026-04-20T14:43:41.949813858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:43:58.184128872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:44:16.389709791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46708"
ts=2026-04-20T14:44:24.210086611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53758"
ts=2026-04-20T14:44:28.186224497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-r6b48-0e9c8f45 10.128.0.79:7946"
level=debug ts=2026-04-20T14:44:39.942607919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:44:54.212293413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47564"
ts=2026-04-20T14:44:58.187806896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"
level=debug ts=2026-04-20T14:45:21.311866953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T14:45:28.190058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-q29ng-9c6e9c9a 10.129.0.66:7946"
ts=2026-04-20T14:45:36.77667531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60572"
ts=2026-04-20T14:45:46.393838395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52774"
ts=2026-04-20T14:45:58.191881313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9994aec9 10.128.0.83:7946"