level=info ts=2026-08-08T11:07:58.062915196Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.4, branch=release-3.7.x, revision=b318f282)"
level=info ts=2026-08-08T11:07:58.062959764Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-08-08T11:07:58.065731345Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-08-08T11:07:58.066786663Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5c5cf9884b-q6flc-14f75e22
level=info ts=2026-08-08T11:07:58.067090075Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-08-08T11:07:58.072768744Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-08T11:07:58.076916951Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-08-08T11:07:58.0769386Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-08T11:07:58.076951238Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=9.863198ms
level=error ts=2026-08-08T11:07:58.076963274Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-08-08T11:07:58.076977843Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-08-08T11:07:58.081156051Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local:9095
ts=2026-08-08T11:07:58.082212222Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-08T11:07:58.082232995Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-08T11:07:58.098815315Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-08-08T11:07:58.098904916Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-08-08T11:07:58.098876293Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-08-08T11:07:58.099082033Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-08-08T11:07:58.099468413Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-08-08T11:07:58.099589567Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-08-08T11:07:58.099627325Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-08-08T11:07:58.099659435Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-08-08T11:07:58.099681606Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-08-08T11:07:58.099707932Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-08-08T11:07:58.099840923Z caller=loki.go:615 msg="Loki started" startup_time=109.504993ms
level=warn ts=2026-08-08T11:07:58.103308069Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-08T11:07:59.887213484Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-08T11:07:59.887244542Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-08T11:08:01.106276744Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-08T11:08:01.975737251Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-08T11:08:01.975765545Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-08T11:08:04.108645783Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.86:9095
ts=2026-08-08T11:08:06.999726906Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-08T11:08:06.999748958Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-08T11:08:18.122229983Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.128.0.68:7946: dial tcp 10.128.0.68:7946: connect: connection refused\n\n"
level=info ts=2026-08-08T11:08:38.44065634Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=40.363672859s
level=info ts=2026-08-08T11:09:14.450894528Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:10:44.450413391Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:12:14.451447541Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:13:44.451935949Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:15:14.451670775Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:16:44.451120031Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:18:14.451446032Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:19:44.451435154Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:21:14.451878473Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:22:44.451497827Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:24:14.4511525Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:25:44.451312574Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:27:14.451397286Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:28:44.449486053Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:30:14.450365751Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:31:44.451331829Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:33:14.450742618Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:34:44.452276564Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:36:14.452576701Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:37:44.451251662Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:39:14.451511246Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:40:44.45117242Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:42:14.45121715Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:43:44.450907006Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:45:14.451566291Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:46:44.450412405Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:48:14.455760508Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:49:44.45132447Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:51:14.45088534Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:52:44.451385701Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:54:14.451321025Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:55:44.451293713Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:57:14.451071394Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T11:58:44.450447Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:00:14.451627907Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:01:44.452316057Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:03:14.450703407Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:04:44.450604179Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:06:14.451428297Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:07:44.451395506Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:09:14.450625739Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:10:44.450726534Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:12:14.451693033Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:13:44.451111484Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:15:14.45228719Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:16:44.451311131Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:18:14.450437047Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:19:44.451146996Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:21:14.450280384Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:22:44.451873203Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:24:14.451075944Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:25:44.450242542Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:27:14.450860235Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:28:44.451106867Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:30:14.452273198Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:31:44.451752863Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:33:14.450715457Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:34:44.451188017Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:36:14.450960612Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:37:44.451200937Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:39:14.450384546Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:40:44.449973066Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:42:14.449874679Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:43:44.45071103Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:45:14.450368449Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:46:44.45050978Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:48:14.450415087Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:49:44.452178111Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:51:14.451929966Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:52:44.450762752Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:54:14.450172136Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-08T12:55:44.451116146Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4