level=info ts=2026-08-14T02:53:14.811683349Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.4, branch=release-3.7.x, revision=b318f282)"
level=info ts=2026-08-14T02:53:14.811722278Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-08-14T02:53:14.813268946Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-08-14T02:53:14.815691844Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5c5cf9884b-fhcg2-a1018be6
level=info ts=2026-08-14T02:53:14.81593768Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-08-14T02:53:14.820259816Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-14T02:53:14.822374634Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-08-14T02:53:14.822408743Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T02:53:14.822480267Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.543832ms
level=error ts=2026-08-14T02:53:14.822541885Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-08-14T02:53:14.822553669Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-08-14T02:53:14.826105536Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T02:53:14.826125702Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T02:53:14.827610503Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-08-14T02:53:14.844785408Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-08-14T02:53:14.844790532Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-08-14T02:53:14.844805448Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-08-14T02:53:14.844814144Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-08-14T02:53:14.844828046Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-08-14T02:53:14.845404015Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-08-14T02:53:14.845443811Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-08-14T02:53:14.845488006Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-08-14T02:53:14.845510436Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-08-14T02:53:14.84555071Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-08-14T02:53:14.845603623Z caller=loki.go:615 msg="Loki started" startup_time=101.204584ms
level=warn ts=2026-08-14T02:53:14.849026992Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-14T02:53:16.499512391Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T02:53:16.499543551Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T02:53:17.852074492Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-08-14T02:53:19.128520752Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T02:53:19.128554168Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T02:53:20.856542716Z caller=worker.go:235 component=querier msg="adding connection" addr=10.129.0.81:9095
ts=2026-08-14T02:53:24.88253672Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T02:53:24.882560969Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T02:53:36.270432317Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.128.0.79:7946: dial tcp 10.128.0.79:7946: connect: connection refused\n\n"
level=info ts=2026-08-14T02:54:00.739035159Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=45.916470564s
level=info ts=2026-08-14T02:55:25.439757401Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T02:56:55.440436918Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T02:58:25.440296818Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T02:59:55.439517335Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:01:25.439818665Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:02:55.440814393Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:04:25.440389035Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:05:55.44161643Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:07:25.441330488Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:08:55.440789276Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:10:25.440248764Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:11:55.439671474Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:13:25.440499523Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:14:55.440833218Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:16:25.439634384Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:17:55.440546718Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:19:25.439973766Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:20:55.440429666Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:22:25.439437355Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:23:55.440152244Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:25:25.439249974Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:26:55.441098183Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:28:25.440158993Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:29:55.441539669Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:31:25.440976516Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:32:55.440218866Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:34:25.440498966Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:35:55.440990412Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:37:25.440032812Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:38:55.439702623Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:40:25.439558526Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:41:55.440804922Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:43:25.439994712Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:44:55.440365999Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:46:25.439873906Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:47:55.439872899Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:49:25.43975334Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:50:55.440248583Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:52:25.440865562Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:53:55.440400373Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:55:25.439031282Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:56:55.441815892Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:58:25.440060783Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:59:55.439935747Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:01:25.43999311Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:02:55.440325917Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:04:25.439206357Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:05:55.442118657Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:07:25.440323581Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:08:55.440448144Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:10:25.440120224Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:11:55.440051715Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:13:25.439646177Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:14:55.441893143Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:16:25.440007365Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:17:55.440835741Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:19:25.439648303Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:20:55.440633489Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:22:25.440130736Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:23:55.440776809Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:25:25.439460196Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:26:55.440384802Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:28:25.440751978Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:29:55.440379281Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:31:25.43966625Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:32:55.439967158Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:34:25.439419044Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:35:55.440678357Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:37:25.439469215Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4