level=info ts=2026-04-17T11:21:10.898287259Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T11:21:10.898315738Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T11:21:10.89948286Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T11:21:10.899763778Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T11:21:10.899825857Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.66 inf=eth0
level=debug ts=2026-04-17T11:21:10.900182055Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T11:21:10.900245465Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.66 inf=eth0
level=info ts=2026-04-17T11:21:10.900270478Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c536c38c
ts=2026-04-17T11:21:10.90032726Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T11:21:10.900338766Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=debug ts=2026-04-17T11:21:10.90035266Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=info ts=2026-04-17T11:21:10.900455638Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T11:21:10.90511153Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T11:21:10.90512632Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T11:21:10.905135598Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.681721ms
level=error ts=2026-04-17T11:21:10.905142234Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T11:21:10.905153101Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T11:21:10.907658926Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T11:21:10.907683544Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T11:21:10.914467394Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-17T11:21:10.914524594Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-17T11:21:10.914530621Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-17T11:21:10.914539056Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-17T11:21:10.914534832Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-17T11:21:10.914652513Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-17T11:21:10.914672294Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-17T11:21:10.914737736Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-17T11:21:10.914745641Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-17T11:21:10.914749254Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T11:21:10.914776939Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T11:21:10.914775681Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T11:21:10.914800627Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T11:21:10.91493909Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T11:21:10.914948025Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T11:21:10.914962481Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T11:21:11.9160202Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-17T11:21:12.10558453Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T11:21:12.105634905Z caller=loki.go:599 msg="Loki started" startup_time=1.273564188s
ts=2026-04-17T11:21:12.218837221Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T11:21:12.218863166Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T11:21:15.269725884Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T11:21:15.269755878Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T11:21:17.106429528Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T11:21:17.106491221Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T11:21:21.42489956Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T11:21:21.424924Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T11:21:24.445175792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T11:21:30.197478448Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T11:21:30.197504165Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T11:21:37.236826393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.42:40018"
level=debug ts=2026-04-17T11:21:40.198311979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T11:21:45.531668687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47802"
ts=2026-04-17T11:21:54.561542465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:45030"
ts=2026-04-17T11:21:57.225179316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33700"
ts=2026-04-17T11:21:59.509780033Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
ts=2026-04-17T11:21:59.510356867Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-04-17T11:21:59.511524188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.42:7946"
ts=2026-04-17T11:21:59.514267886Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-04-17T11:21:59.514512156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.64:7946"
ts=2026-04-17T11:21:59.515221326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-04-17T11:21:59.515249973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:54970"
ts=2026-04-17T11:21:59.515748087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
level=info ts=2026-04-17T11:21:59.516251426Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=48.611091944s
ts=2026-04-17T11:21:59.907147137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ff794d8b 10.129.0.67:7946"
level=debug ts=2026-04-17T11:22:02.096908344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T11:22:28.782016862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T11:22:29.909016507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qpwnq-d44656d1 10.130.0.42:7946"
ts=2026-04-17T11:22:45.533987858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:55766"
ts=2026-04-17T11:22:54.022496968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49382"
ts=2026-04-17T11:22:59.910979198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bffrl-13b83709 10.129.0.64:7946"
level=debug ts=2026-04-17T11:23:00.111042556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T11:23:00.1127226Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T11:23:01.335118389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.42:41112"
ts=2026-04-17T11:23:15.535542775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:36372"
ts=2026-04-17T11:23:29.912373717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bffrl-13b83709 10.129.0.64:7946"
level=debug ts=2026-04-17T11:23:41.255643473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"