level=info ts=2026-08-14T03:22:54.156620063Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.4, branch=release-3.7.x, revision=b318f282)"
level=info ts=2026-08-14T03:22:54.156641356Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-08-14T03:22:54.157942366Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-08-14T03:22:54.158853761Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c77b4c71
level=info ts=2026-08-14T03:22:54.159062516Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-08-14T03:22:54.162476395Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-08-14T03:22:54.162489542Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-08-14T03:22:54.162499679Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=3.438835ms
level=error ts=2026-08-14T03:22:54.162508412Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-08-14T03:22:54.16251868Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-08-14T03:22:54.164225424Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:22:54.16423802Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-08-14T03:22:55.844981144Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:22:55.845005193Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:22:56.478241703Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-08-14T03:22:56.478250457Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-08-14T03:22:56.478305516Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-08-14T03:22:56.478313513Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-08-14T03:22:56.478527574Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-08-14T03:22:56.47860305Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-08-14T03:22:56.478593398Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-08-14T03:22:56.478620997Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-08-14T03:22:56.478785028Z caller=compactor.go:420 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-08-14T03:22:56.478803542Z caller=compactor.go:424 msg="compactor is JOINING in the ring"
level=info ts=2026-08-14T03:22:57.479934418Z caller=compactor.go:434 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-08-14T03:22:57.47996139Z caller=compactor.go:438 msg="compactor is ACTIVE in the ring"
level=info ts=2026-08-14T03:22:57.479990858Z caller=loki.go:615 msg="Loki started" startup_time=3.383588602s
ts=2026-08-14T03:22:58.430156575Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:22:58.43019148Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:23:02.480474968Z caller=compactor.go:497 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-08-14T03:23:02.48055062Z caller=tables_manager.go:70 msg="waiting 15m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-08-14T03:23:04.610537693Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-08-14T03:23:04.610566304Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-08-14T03:23:16.470588201Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.308065185s
level=info ts=2026-08-14T03:23:59.798737914Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:25:29.800732383Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:26:59.800294034Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:28:29.799615433Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:29:59.800396221Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:31:29.799699097Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:32:59.800408268Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:34:29.799014987Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:35:59.799734008Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:37:29.79832116Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:38:02.48084534Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=info ts=2026-08-14T03:38:02.488511313Z caller=tables_manager.go:148 msg="compactor started"
level=info ts=2026-08-14T03:38:02.488533169Z caller=marker.go:181 msg="mark processor started" workers=150 delay=2h0m0s
level=info ts=2026-08-14T03:38:02.488533695Z caller=expiration.go:82 msg="overall smallest retention period 1786592282.488, default smallest retention period 1786592282.488"
level=info ts=2026-08-14T03:38:02.488592641Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:38:59.799323821Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:39:02.489117343Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:40:02.488984925Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:40:29.801773205Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:41:02.489220359Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:41:59.799853687Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:42:02.489125001Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:43:02.488628252Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:43:29.800733354Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:44:02.489542389Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:44:59.800230871Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:45:02.489010825Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:46:02.48892373Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:46:29.799359392Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:47:02.489477085Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:47:59.799427319Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:48:02.488864019Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:49:02.488981734Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:49:29.799091038Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:50:02.489222819Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:50:59.799989495Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:51:02.489622014Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:52:02.489038209Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:52:29.800104171Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:53:02.489452697Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:53:59.800424094Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:54:02.488812931Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:55:02.488713928Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:55:29.79879585Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:56:02.488931947Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:56:59.801599349Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:57:02.489204362Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:58:02.489117111Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:58:29.798924924Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T03:59:02.489205879Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T03:59:59.798756032Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:00:02.489270444Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:00:32.491284445Z caller=expiration.go:82 msg="overall smallest retention period 1786593632.491, default smallest retention period 1786593632.491"
level=info ts=2026-08-14T04:01:02.488652751Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:01:29.798870424Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:02:02.489598056Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:02:59.800017631Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:03:02.489110258Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:04:02.489050924Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:04:29.801395852Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:05:02.489081301Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:05:59.798874338Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:06:02.489643132Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:07:02.489617341Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:07:29.800074988Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:08:02.48945304Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:08:59.800502188Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:09:02.489622931Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:10:02.488786477Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:10:29.799797185Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:11:02.489132025Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:11:59.798578695Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:12:02.488859642Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:13:02.488941384Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:13:29.799498173Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:14:02.489160007Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:14:59.799595838Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:15:02.488799119Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:16:02.489407944Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:16:29.799514125Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:17:02.489567515Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:17:59.799853638Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:18:02.488924342Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:19:02.489539893Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:19:29.799643365Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:20:02.489328626Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:20:59.799999043Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:21:02.488991018Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:22:02.489110248Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:22:29.798825397Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:23:02.488737947Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:23:02.490904452Z caller=expiration.go:82 msg="overall smallest retention period 1786594982.49, default smallest retention period 1786594982.49"
level=info ts=2026-08-14T04:23:59.799092197Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:24:02.488752397Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:25:02.489591715Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:25:29.798835987Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:26:02.488943005Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:26:59.799461536Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:27:02.489138099Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:28:02.489636133Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:28:29.801333614Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:29:02.489059747Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:29:59.799440527Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:30:02.489157939Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:31:02.489411487Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:31:29.800858946Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:32:02.48878035Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:32:59.800346843Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:33:02.489280439Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:34:02.488999511Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:34:29.799666299Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:35:02.488802466Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:35:59.800104587Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:36:02.489125669Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:37:02.488995331Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:37:29.799930959Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:38:02.489648973Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:38:59.800105883Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:39:02.488970953Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:40:02.489224628Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:40:29.799737012Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:41:02.488959506Z caller=marker.go:206 msg="no marks file found"
level=info ts=2026-08-14T04:41:59.798890184Z caller=memberlist_client.go:653 phase=periodic_rejoin msg="re-joined memberlist cluster" reached_nodes=4
level=info ts=2026-08-14T04:42:02.489425147Z caller=marker.go:206 msg="no marks file found"