level=info ts=2026-06-29T05:38:05.423787238Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-29T05:38:05.423832634Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-29T05:38:05.426492959Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-29T05:38:05.427452296Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-distributor-7696f46544-d4nrd-a8def74a
level=info ts=2026-06-29T05:38:05.427677269Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-06-29T05:38:05.433458586Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T05:38:05.433471791Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T05:38:05.433481395Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.806125ms
level=error ts=2026-06-29T05:38:05.433489555Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-29T05:38:05.433499906Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-29T05:38:05.435706785Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:05.435719452Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T05:38:05.441250667Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-29T05:38:05.44125047Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-29T05:38:05.441273474Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-29T05:38:05.441286142Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-29T05:38:05.441562418Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-29T05:38:05.441597696Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T05:38:05.441624051Z caller=module_service.go:82 msg=starting module=distributor
level=info ts=2026-06-29T05:38:05.441662703Z caller=basic_lifecycler.go:324 component=distributor msg="instance not found in the ring" instance=loki-distributor-7696f46544-d4nrd ring=distributor
level=info ts=2026-06-29T05:38:05.441863643Z caller=loki.go:615 msg="Loki started" startup_time=75.92344ms
ts=2026-06-29T05:38:07.286265352Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:07.286289262Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T05:38:09.356872305Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:09.356896731Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T05:38:16.616226084Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T05:38:16.616244973Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T05:38:19.018464705Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 26.28311ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T05:38:21.494711111Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 22.874139ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T05:38:22.202613502Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 22.303175ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T05:38:22.444408555Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 24.103193ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T05:38:23.877165025Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.576936ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T05:38:24.236041357Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 23.001462ms Response: \"empty ring\\n\""
level=info ts=2026-06-29T05:38:30.83473878Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=25.401233782s
level=error ts=2026-06-29T05:38:41.973121436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD76AF1C2B3BAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:42:41.378697086Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD76E6D9DC435B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:46:31.409461345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD771C68C31F03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:50:47.166686896Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7757F5147D66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:55:02.054268549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD77934D90BBC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T05:58:59.684710621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD77CAA171BD18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:02:33.462602495Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD77FC6799724F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:06:06.930265588Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD782E1B43916B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:09:56.174330476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD78637B455B8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:14:23.492731901Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD78A1B8B17D3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:18:23.336185221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD78D990799A98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:22:05.440625518Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD790D46EF3212, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:26:10.359765713Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD79464D405E1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:30:21.038233671Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7980AADA40F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:34:42.763417499Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD79BD9AE3E07B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:38:39.454224804Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD79F4B6C2A5B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:41:48.960070822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7A20D63032D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:46:30.522671717Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7A6264A0FEA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:51:09.336237291Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7AA34F3640B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:55:08.094836244Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7ADAE6554C6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T06:59:00.047868678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B10E7D06B2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:02:46.956218296Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B45BC9A53B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:07:04.513428871Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7B81B4351DC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T07:10:51.32977907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BD7BB68384BACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"