level=info ts=2026-08-14T03:11:57.787859173Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.4, branch=release-3.7.x, revision=b318f282)"
level=info ts=2026-08-14T03:11:57.787888637Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-08-14T03:11:57.792111761Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-08-14T03:11:57.79307853Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5c5cf9884b-xh896-84fea38a
level=info ts=2026-08-14T03:11:57.793301702Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-08-14T03:11:57.795521537Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-14T03:11:57.798300077Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-08-14T03:11:57.798327281Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T03:11:57.798341185Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.041357ms
level=error ts=2026-08-14T03:11:57.798356355Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-08-14T03:11:57.798377599Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-08-14T03:11:57.80036752Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:11:57.800388937Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-08-14T03:11:57.800873409Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:11:57.800901966Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:11:57.801091401Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-08-14T03:11:57.817191663Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-08-14T03:11:57.817206561Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-08-14T03:11:57.817261169Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-08-14T03:11:57.817279838Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-08-14T03:11:57.817320389Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-08-14T03:11:57.817506317Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-08-14T03:11:57.817542586Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-08-14T03:11:57.817579868Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-08-14T03:11:57.817592792Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-08-14T03:11:57.817613374Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-08-14T03:11:57.817668774Z caller=loki.go:615 msg="Loki started" startup_time=104.354227ms
level=warn ts=2026-08-14T03:11:57.82096371Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-14T03:11:59.523239697Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:11:59.523276651Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T03:12:00.823745054Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=error ts=2026-08-14T03:12:02.799079138Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:12:02.799105149Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-08-14T03:12:03.141695772Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:12:03.141721062Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:12:03.828198631Z caller=worker.go:235 component=querier msg="adding connection" addr=10.129.0.71:9095
ts=2026-08-14T03:12:08.611820573Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:12:08.611846993Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:12:17.043974616Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=19.245590318s
level=info ts=2026-08-14T03:13:13.562701006Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:14:43.563313227Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:16:13.562886099Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:17:43.562927424Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:19:13.562818922Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:20:43.563093433Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:22:13.562996527Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:23:43.562792494Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:25:13.564011518Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:26:43.562938431Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:28:13.563449769Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:29:43.562758012Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:31:13.563084747Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:32:43.562948353Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:34:13.562888684Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:35:43.563827096Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:37:13.562808808Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:38:43.563072766Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:40:13.563594358Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:41:43.563142729Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:43:13.562730613Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:44:43.56322141Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:46:13.563437203Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:47:43.563417621Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:49:13.563802947Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:50:43.562965483Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:52:13.562944476Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:53:43.563933026Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:55:13.563952416Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:56:43.56357436Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:58:13.563448143Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:59:43.56296867Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:01:13.563012035Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:02:43.5629756Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:04:13.563516246Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:05:43.562746023Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:07:13.563377509Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:08:43.564237017Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:10:13.562880255Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:11:43.563241219Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:13:13.563427496Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:14:43.563235298Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:16:13.563469612Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:17:43.562667925Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:19:13.563491361Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:20:43.562292365Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:22:13.5623243Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:23:43.562824744Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:25:13.562938263Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:26:43.563630498Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:28:13.563061192Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:29:43.562840485Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:31:13.56244341Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:32:43.563046508Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:34:13.562404903Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:35:43.563250342Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:37:13.563087676Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:38:43.563768052Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:40:13.562808155Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:41:43.562897167Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:43:13.562724132Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:44:43.563124341Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:46:13.563727482Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:47:43.56321107Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:49:13.563026306Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:50:43.563193909Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:52:13.563098801Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:53:43.563248658Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:55:13.563786931Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:56:43.563200866Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:58:13.56393985Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:59:43.563460041Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T05:01:13.563437864Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T05:02:43.561985301Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T05:04:13.562814498Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T05:05:43.563411832Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T05:07:13.56359907Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4