level=info ts=2026-04-16T19:45:19.63729887Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T19:45:19.637332426Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-16T19:45:19.637639551Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-16T19:45:19.638284459Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T19:45:19.638633374Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T19:45:19.6387044Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.79 inf=eth0
level=info ts=2026-04-16T19:45:19.639180108Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-5c4e24c3
ts=2026-04-16T19:45:19.639269887Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T19:45:19.639288559Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.79 advertisePort=7946
level=debug ts=2026-04-16T19:45:19.639302023Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.79 advertisePort=7946
level=info ts=2026-04-16T19:45:19.639406323Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-16T19:45:19.641402641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.56:11211 count=1
level=info ts=2026-04-16T19:45:19.642844454Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-16T19:45:19.642918509Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-16T19:45:19.643897411Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-16T19:45:19.643915147Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-16T19:45:19.643913643Z
level=debug ts=2026-04-16T19:45:19.643927974Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-16T19:45:19.643940721Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-16T19:45:19.64395416Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-04-16T19:45:19.643958445Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-16T19:45:19.643957911Z
level=debug ts=2026-04-16T19:45:19.643962319Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-16T19:45:19.64504696Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T19:45:19.645135061Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.79 inf=eth0
ts=2026-04-16T19:45:19.646665045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.53:7946"
ts=2026-04-16T19:45:19.649689686Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.55:7946: dial tcp 10.128.0.55:7946: connect: connection refused"
ts=2026-04-16T19:45:19.651485006Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused"
ts=2026-04-16T19:45:19.652603975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-04-16T19:45:19.654776317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-16T19:45:19.65597665Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
level=info ts=2026-04-16T19:45:19.655985711Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=16.582113ms
level=info ts=2026-04-16T19:45:19.656001501Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-16T19:45:19.658433557Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-16T19:45:19.65850253Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-16T19:45:19.658520748Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-16T19:45:19.658529321Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-16T19:45:19.65854197Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-16T19:45:19.658535773Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-16T19:45:19.65855022Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T19:45:19.658547953Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-16T19:45:19.658561672Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-16T19:45:19.658565475Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-16T19:45:19.658565864Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-16T19:45:19.65857164Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=info ts=2026-04-16T19:45:19.658586871Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-16T19:45:19.658898072Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-16T19:45:19.65890484Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-16T19:45:19.658905794Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-16T19:45:19.658918868Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-16T19:45:19.658938816Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-16T19:45:19.659013951Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-04-16T19:45:19.659018109Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-16T19:45:19.659036369Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-16T19:45:19.659040603Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-16T19:45:19.659043585Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-04-16T19:45:19.659046385Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-16T19:45:19.659059308Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-04-16T19:45:19.659413027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.53:7946"
ts=2026-04-16T19:45:19.661771464Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.55:7946: dial tcp 10.128.0.55:7946: connect: connection refused"
ts=2026-04-16T19:45:19.662716318Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused"
ts=2026-04-16T19:45:19.663726718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-04-16T19:45:19.665038782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-16T19:45:19.665587399Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
level=info ts=2026-04-16T19:45:19.665602934Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=9.597238ms
level=info ts=2026-04-16T19:45:19.737825836Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=78.766688ms errors=false
level=info ts=2026-04-16T19:45:19.737858275Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-16T19:45:24.41679797Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=4.757738331s errors=false
level=info ts=2026-04-16T19:45:24.416831916Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-16T19:45:24.416866002Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=4.757807189s
level=info ts=2026-04-16T19:45:24.416884227Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-16T19:45:24.416937202Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=21.302904551s
level=info ts=2026-04-16T19:45:24.416941016Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T19:45:24.416981902Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-04-16T19:45:24.41701632Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-16T19:45:24.417054683Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-16T19:45:24.417077309Z caller=loki.go:599 msg="Loki started" startup_time=4.836299156s
level=warn ts=2026-04-16T19:45:29.418060729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T19:45:34.417791932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T19:45:34.638796314Z caller=logging.go:137 msg="GET /ready (503) 93.263µs"
level=warn ts=2026-04-16T19:45:39.41805186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T19:45:44.417493461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T19:45:44.637139802Z caller=logging.go:137 msg="GET /ready (503) 92.395µs"
level=warn ts=2026-04-16T19:45:49.417135355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T19:45:50.532468151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42724"
level=info ts=2026-04-16T19:45:54.4180427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-16T19:45:54.418078685Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-16T19:45:54.418088748Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=info ts=2026-04-16T19:45:54.418098468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T19:45:54.418127642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T19:45:59.418153392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T19:46:00.469492472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:35510"
level=warn ts=2026-04-16T19:46:04.417654216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T19:46:09.417409045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T19:46:10.131927408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:46:13.52654682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"