level=info ts=2026-06-29T14:45:15.644639975Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-29T14:45:15.644667724Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-29T14:45:15.646620854Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-29T14:45:15.647466823Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-distributor-7696f46544-9xr6g-8e084b79
level=info ts=2026-06-29T14:45:15.647730383Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-06-29T14:45:15.652201308Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T14:45:15.652214178Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T14:45:15.652224961Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.497711ms
level=error ts=2026-06-29T14:45:15.652233206Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-29T14:45:15.65224403Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-29T14:45:15.655690107Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T14:45:15.655711623Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T14:45:15.66217323Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-29T14:45:15.662195821Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-29T14:45:15.66222149Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-29T14:45:15.662258273Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-29T14:45:15.662548872Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-29T14:45:15.662585544Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T14:45:15.662665703Z caller=module_service.go:82 msg=starting module=distributor
level=info ts=2026-06-29T14:45:15.662697363Z caller=basic_lifecycler.go:324 component=distributor msg="instance not found in the ring" instance=loki-distributor-7696f46544-9xr6g ring=distributor
level=info ts=2026-06-29T14:45:15.662741126Z caller=ring.go:390 component=distributor msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T14:45:15.662872859Z caller=loki.go:615 msg="Loki started" startup_time=89.412125ms
ts=2026-06-29T14:45:16.826815393Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T14:45:16.826836164Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T14:45:20.710810066Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T14:45:20.71083193Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T14:45:24.727219997Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 25.916565ms Response: \"empty ring\\n\""
ts=2026-06-29T14:45:25.245474414Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T14:45:25.245499331Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T14:45:25.751933125Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 22.651366ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T14:45:30.783892047Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 21.558783ms Response: \"empty ring\\n\""
ts=2026-06-29T14:45:36.625025376Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T14:45:36.625050174Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T14:45:38.377914228Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.906474ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T14:45:39.981949444Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 25.515475ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T14:45:46.241537253Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.541356ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T14:45:46.830786757Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.930096ms Response: \"empty ring\\n\""
level=error ts=2026-06-29T14:45:48.933448095Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD948A3B7C6CC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T14:45:50.010685935Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.324176ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T14:45:51.835872887Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.490055ms Response: \"empty ring\\n\""
level=info ts=2026-06-29T14:46:04.292058368Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=48.639809999s
level=error ts=2026-06-29T14:49:32.806403147Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD94BE5B59EA14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T14:53:52.004094568Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD94FAB4BCA8AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T14:58:05.69564953Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9535C5EF36E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:02:18.543536362Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9570A4D69BD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:06:24.526573536Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD95A9EA924FC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:09:50.097908855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD95D9C793FF11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:13:54.785059728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9612C0124AAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:17:37.696926425Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9646A6A72438, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:21:29.303449268Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD967C9379E3BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:25:28.445037154Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD96B4416CAF2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:29:25.843231191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD96EB877686CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:33:27.486952081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9723CA8C30D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:37:26.473691593Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD975B6F45BC24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:41:22.654290276Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD97926CBA86AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:45:31.360701787Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD97CC54C7F403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:49:11.474567397Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD97FF94987D25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:53:04.687877843Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD9835E1313DE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T15:56:45.066737612Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD986930CB6F3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T16:00:46.482410582Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD98A1664A1CCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T16:04:58.655069203Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD98DC1CF445CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T16:08:44.034796044Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD991096A027C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T16:12:25.466506118Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD994424FE5BD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T16:16:42.229823016Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD997FED46541B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"