level=info ts=2026-08-22T02:09:17.899310537Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.4, branch=release-3.7.x, revision=b318f282)"
level=info ts=2026-08-22T02:09:17.899337533Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-08-22T02:09:17.901129735Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-08-22T02:09:17.902349188Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5c5cf9884b-ngcrv-42cad3ef
level=info ts=2026-08-22T02:09:17.902557161Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-08-22T02:09:17.904521783Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-22T02:09:17.907457485Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-08-22T02:09:17.907478689Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-22T02:09:17.907500027Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.937047ms
level=error ts=2026-08-22T02:09:17.907523287Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-08-22T02:09:17.907560794Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-08-22T02:09:17.909764463Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:17.909778435Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-08-22T02:09:17.91045093Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:17.910468179Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-08-22T02:09:17.911215301Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-08-22T02:09:17.925311388Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-08-22T02:09:17.925323966Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-08-22T02:09:17.925357455Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-08-22T02:09:17.925363407Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-08-22T02:09:17.925385102Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-08-22T02:09:17.925602711Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-08-22T02:09:17.92563775Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-08-22T02:09:17.925670436Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-08-22T02:09:17.925685548Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-08-22T02:09:17.925711741Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-08-22T02:09:17.925755761Z caller=loki.go:615 msg="Loki started" startup_time=97.394927ms
level=warn ts=2026-08-22T02:09:17.927700883Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-22T02:09:19.671133973Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:19.671157395Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-22T02:09:20.930269996Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-22T02:09:21.919002717Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:21.919026058Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-08-22T02:09:22.908527562Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:22.908571242Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-08-22T02:09:23.933414171Z caller=worker.go:235 component=querier msg="adding connection" addr=10.128.0.77:9095
ts=2026-08-22T02:09:27.205114325Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:27.205136527Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-08-22T02:09:41.999281597Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-22T02:09:41.99930466Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-22T02:10:10.140806385Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=52.233238719s
level=info ts=2026-08-22T02:10:33.354607856Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:12:03.357438926Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:13:33.35644727Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:15:03.363833529Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:16:33.356570982Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:18:03.35670176Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:19:33.355487748Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:21:03.356089977Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:22:33.356170646Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:24:03.355752578Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:25:33.356574032Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:27:03.356863791Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:28:33.356266091Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:30:03.363186251Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:31:33.354991799Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:33:03.356754089Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:34:33.3556912Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:36:03.356177275Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:37:33.355579467Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:39:03.355179762Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:40:33.355705572Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:42:03.355317964Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:43:33.356088614Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:45:03.355824104Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:46:33.355338586Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:48:03.355380875Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:49:33.355059349Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:51:03.356215036Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:52:33.355119609Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:54:03.355706762Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:55:33.35632707Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:57:03.356086423Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T02:58:33.354595429Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:00:03.356422436Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:01:33.355501615Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:03:03.356347467Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:04:33.355438539Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:06:03.355084095Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:07:33.35550694Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:09:03.355591022Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:10:33.355436797Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:12:03.355857496Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:13:33.355268384Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:15:03.35528503Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:16:33.355483117Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:18:03.355380684Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:19:33.356522111Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:21:03.355838561Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:22:33.355885272Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:24:03.354804237Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:25:33.355488006Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:27:03.35604775Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:28:33.35488351Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:30:03.35580756Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:31:33.355735656Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:33:03.355391881Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:34:33.356015065Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:36:03.355918751Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:37:33.356048318Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:39:03.356205956Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:40:33.355357995Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:42:03.355980241Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:43:33.356795373Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:45:03.355459625Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:46:33.35569736Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:48:03.355437579Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:49:33.355435917Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:51:03.356112039Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:52:33.355174748Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-22T03:54:03.355089792Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4