level=info ts=2026-06-29T22:19:49.811458731Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-29T22:19:49.811514768Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-29T22:19:49.814953754Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-29T22:19:49.815859993Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-distributor-7696f46544-dwc99-a0875bdb
level=info ts=2026-06-29T22:19:49.816207866Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-06-29T22:19:49.825927198Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T22:19:49.825956508Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T22:19:49.825989838Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=9.785394ms
level=error ts=2026-06-29T22:19:49.826003238Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-29T22:19:49.826019955Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-29T22:19:49.831918985Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:49.831948271Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T22:19:49.834894267Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-29T22:19:49.835342882Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-29T22:19:49.836209825Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-29T22:19:49.836378265Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-29T22:19:49.836420666Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-29T22:19:49.83647692Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T22:19:49.836627302Z caller=module_service.go:82 msg=starting module=distributor
level=info ts=2026-06-29T22:19:49.836684941Z caller=basic_lifecycler.go:324 component=distributor msg="instance not found in the ring" instance=loki-distributor-7696f46544-dwc99 ring=distributor
level=info ts=2026-06-29T22:19:49.83768962Z caller=loki.go:615 msg="Loki started" startup_time=92.657504ms
ts=2026-06-29T22:19:51.430138852Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:51.430168309Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T22:19:54.056187709Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:54.056221684Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T22:19:59.342106304Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 35.469886ms Response: \"empty ring\\n\""
level=info ts=2026-06-29T22:19:59.676727943Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=9.850701109s
level=warn ts=2026-06-29T22:20:01.792242772Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 25.704004ms Response: \"empty ring\\n\""
level=warn ts=2026-06-29T22:20:06.746762116Z caller=logging.go:144 orgID=kubearchive msg="POST /loki/api/v1/push (500) 26.121766ms Response: \"empty ring\\n\""
level=error ts=2026-06-29T22:20:27.100790801Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAD596CF38605, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:23:47.381508966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAD880E9BF6D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:28:01.773568952Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDADC3499177B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:32:02.156776923Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDADFB41869ED7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:35:32.498229178Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE2C3AD980F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:39:17.161625747Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE6089D4BE54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:43:41.089505395Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE9DFD27FAB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:47:07.474527425Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAECE0AAB3878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:50:54.95062756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF03014C975A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:55:02.954361482Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF3CBF7921D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T22:58:57.092089452Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF734326E94C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:02:53.506982789Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAFAA4E981324, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:06:50.459236385Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAFE17A0CBB7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:10:58.737252298Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB01B4892D98B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:15:23.090221832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB058D53CD542, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:19:28.350448661Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB091EFE24196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:23:36.901149997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB0CBCEA8F80C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:27:26.318668962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB1013902A49F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:31:15.124588997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB1367EE7B634, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:35:40.201145995Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB17436B2FA41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:39:48.08373595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB1ADEDA87A17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:43:48.731233965Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB1E5F55CAD53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:47:31.94716933Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB219EE121790, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:51:04.924954547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB24B8489432A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:55:05.685781657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2839300154E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-29T23:59:04.147340969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2BB186B091E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=error ts=2026-06-30T00:02:59.287247643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2F1D7D87384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"