level=info ts=2026-06-29T05:38:07.585314537Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-29T05:38:07.585337886Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-29T05:38:07.586328839Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-29T05:38:07.58815839Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-f94f578b6-vfkk2-fb4003f1
level=info ts=2026-06-29T05:38:07.588443715Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-06-29T05:38:07.591946582Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-06-29T05:38:07.593928917Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T05:38:07.593999162Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T05:38:07.594024822Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.583256ms
level=error ts=2026-06-29T05:38:07.594034548Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-29T05:38:07.594046488Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-29T05:38:07.596472423Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:07.596486478Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T05:38:07.597990714Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-06-29T05:38:07.611744257Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-06-29T05:38:07.611757446Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-29T05:38:07.611759846Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-29T05:38:07.611768031Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-29T05:38:07.611802714Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-29T05:38:07.612010491Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-29T05:38:07.612040222Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T05:38:07.612081304Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-06-29T05:38:07.612097122Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-06-29T05:38:07.612116429Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-06-29T05:38:07.612169005Z caller=loki.go:615 msg="Loki started" startup_time=86.356775ms
level=info ts=2026-06-29T05:38:07.615465953Z caller=worker.go:235 component=querier msg="adding connection" addr=172.30.160.132:9095
ts=2026-06-29T05:38:09.047415019Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:09.047440964Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T05:38:13.015170233Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:13.015198536Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T05:38:18.684728477Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:18.684754698Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T05:38:29.98111313Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.387061319s
level=error ts=2026-06-29T05:38:42.083371279Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD76AF22B9F73B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:42:23.1373807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD76E29A9321A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:47:09.235087919Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD772537540371, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:51:38.257561142Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7763DA503791, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:55:30.928784319Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD779A069AD4B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:59:25.178013907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD77D090F10C06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:03:03.452532069Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7803631DF56C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:07:14.817590541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD783DE9A4ECD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:11:51.976584306Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD787E719CA508, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:15:54.375676046Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD78B6E1B9764F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:20:01.602329797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD78F071920341, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:24:03.154108786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7928AF2EAD75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:28:04.075749318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7960C73BB40F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:32:28.105993288Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD799E40A9319A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:36:04.216857226Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD79D091DE48A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:40:15.194891728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7A0B0153D389, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:44:22.681744869Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7A44A0B173CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:48:14.389198612Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7A7A9389248D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:51:50.329388135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7AACDA92A361, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:55:49.083235286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7AE47168F09D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:00:03.367010528Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B1FA5E7FB2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:03:57.675646457Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B5633CA6474, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:07:30.325489185Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B87B6B4D67C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:11:17.696273847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7BBCA70EEFE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"