level=info ts=2026-02-16T09:59:19.734787341Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T09:59:19.734821312Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T09:59:19.736062677Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T09:59:19.736856502Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T09:59:19.736933119Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.75 inf=eth0
level=info ts=2026-02-16T09:59:19.737387353Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-9n5vp-f59fd426
ts=2026-02-16T09:59:19.737466167Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T09:59:19.737481485Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.75 advertisePort=7946
level=debug ts=2026-02-16T09:59:19.73750355Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.75 advertisePort=7946
level=info ts=2026-02-16T09:59:19.737642781Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-02-16T09:59:19.739205905Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.72:11211 count=1
ts=2026-02-16T09:59:19.74555978Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=info ts=2026-02-16T09:59:19.745579216Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=warn ts=2026-02-16T09:59:19.745590902Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.950139ms
level=error ts=2026-02-16T09:59:19.74560046Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T09:59:19.745614233Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T09:59:19.746289863Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T09:59:19.746630801Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T09:59:19.746670278Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
ts=2026-02-16T09:59:19.749456443Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:19.749479657Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=debug ts=2026-02-16T09:59:19.762084954Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=info ts=2026-02-16T09:59:19.76208814Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T09:59:19.762107751Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=info ts=2026-02-16T09:59:19.762109578Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T09:59:19.762119237Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=info ts=2026-02-16T09:59:19.762124678Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T09:59:19.762119891Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T09:59:19.762142384Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T09:59:19.762154011Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=info ts=2026-02-16T09:59:19.762145254Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-02-16T09:59:19.762150484Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T09:59:19.762169768Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T09:59:19.762340543Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T09:59:19.762374264Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-02-16T09:59:19.76234624Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T09:59:19.762391266Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T09:59:19.762488765Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T09:59:19.762518756Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T09:59:19.762523232Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T09:59:19.762528183Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T09:59:19.762537105Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T09:59:19.762540819Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T09:59:19.762543421Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T09:59:19.762546184Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T09:59:19.762548653Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T09:59:19.762554317Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-02-16T09:59:19.76255707Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T09:59:19.762559797Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-02-16T09:59:19.762562151Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T09:59:19.762564488Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=info ts=2026-02-16T09:59:19.762568116Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T09:59:19.762945297Z caller=loki.go:599 msg="Loki started" startup_time=96.708365ms
level=warn ts=2026-02-16T09:59:19.765934776Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T09:59:21.493608776Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:21.493630844Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=warn ts=2026-02-16T09:59:22.768616888Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T09:59:23.782179947Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:23.78220462Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=info ts=2026-02-16T09:59:25.770769967Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.76:9095
ts=2026-02-16T09:59:28.23840603Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:28.238429214Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=debug ts=2026-02-16T09:59:35.571563542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T09:59:37.994403996Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-02-16T09:59:37.99533237Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-02-16T09:59:37.996301787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.74:7946"
level=info ts=2026-02-16T09:59:37.996854962Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=18.251234583s
ts=2026-02-16T09:59:50.591640939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41696"
level=debug ts=2026-02-16T09:59:51.34712855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T09:59:53.70903912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:41068"
ts=2026-02-16T09:59:59.526118634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-ftqjd-3f83b3d8 10.130.0.74:7946"
level=debug ts=2026-02-16T10:00:12.700961405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T10:00:19.741051402Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.72:11211 count=1
ts=2026-02-16T10:00:29.527702551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-12180ce6 10.129.0.76:7946"
level=debug ts=2026-02-16T10:00:43.268591438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T10:00:48.412052622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:58476"