level=info ts=2026-06-30T11:05:31.146780979Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-30T11:05:31.146821187Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-30T11:05:31.148305988Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-30T11:05:31.149148651Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-distributor-7696f46544-vc45c-ba14b81d
level=info ts=2026-06-30T11:05:31.149404459Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-06-30T11:05:31.155376173Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-30T11:05:31.155393167Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T11:05:31.155403921Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.001198ms
level=error ts=2026-06-30T11:05:31.15541188Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-30T11:05:31.155424383Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-30T11:05:31.15843277Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T11:05:31.158453806Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-30T11:05:31.165465027Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-30T11:05:31.165472181Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-30T11:05:31.165479409Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-30T11:05:31.165516033Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-30T11:05:31.16576574Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-30T11:05:31.165819418Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-30T11:05:31.165845329Z caller=module_service.go:82 msg=starting module=distributor
level=info ts=2026-06-30T11:05:31.165885017Z caller=basic_lifecycler.go:324 component=distributor msg="instance not found in the ring" instance=loki-distributor-7696f46544-vc45c ring=distributor
level=info ts=2026-06-30T11:05:31.165922784Z caller=ring.go:390 component=distributor msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-30T11:05:31.166065283Z caller=loki.go:615 msg="Loki started" startup_time=90.970287ms
ts=2026-06-30T11:05:32.784431393Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T11:05:32.784458247Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-30T11:05:36.374550868Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T11:05:36.374573403Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-30T11:05:42.633787676Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T11:05:42.633821551Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T11:05:43.436546372Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.539853ms Response: \"empty ring\\n\""
level=warn ts=2026-06-30T11:05:45.824018564Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 25.041651ms Response: \"empty ring\\n\""
level=warn ts=2026-06-30T11:05:48.694347498Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 21.979429ms Response: \"empty ring\\n\""
level=info ts=2026-06-30T11:05:51.793346254Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.63791672s
level=warn ts=2026-06-30T11:05:52.334851568Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 21.825491ms Response: \"empty ring\\n\""
level=warn ts=2026-06-30T11:05:52.559569133Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 24.134296ms Response: \"empty ring\\n\""
level=error ts=2026-06-30T11:06:08.835548866Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD72211B7191B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:09:22.300747608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD74F1D240873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:13:27.028296282Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD7881808FD90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:17:47.453826801Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD7C4BA9A4AE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:21:46.817121765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD7FC75C6D518, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:25:30.451336515Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD83087694EE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:29:40.302774653Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD86AB3B5E577, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:33:34.980152467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD8A157947588, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:37:12.329876175Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD8D3F29F7E56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:41:02.693058315Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD90995592643, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:45:12.383731326Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD943B8120D24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:49:23.953663874Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD97E4ACF522C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:53:22.114623559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD9B5BE4EC8B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T11:57:33.707145771Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDD9F052645691, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:00:53.834308456Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDA1EEAE63D97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:04:57.436470563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDA57A2B8366E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:08:36.531449186Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDA8AA5CBCDD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:12:38.72549552Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDAC309AEF63D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:16:22.549115934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDAF7269C62B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:20:46.248577267Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDB348C51B442, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:24:29.381907498Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDB68801C0CAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:28:50.24441314Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDBA53CB9676F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:33:20.145003786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDBE4140E4882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:37:40.008702307Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDC2095221B58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T12:40:50.693487935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDDC4CFAD6D22C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"