level=info ts=2026-02-16T09:59:19.725641141Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T09:59:19.725668735Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T09:59:19.726436344Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T09:59:19.726756877Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T09:59:19.726834144Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.77 inf=eth0
level=debug ts=2026-02-16T09:59:19.727715045Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T09:59:19.727773165Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.77 inf=eth0
level=info ts=2026-02-16T09:59:19.728069974Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b8901257
ts=2026-02-16T09:59:19.728170196Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T09:59:19.728188247Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.77 advertisePort=7946
level=debug ts=2026-02-16T09:59:19.728215197Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.77 advertisePort=7946
level=info ts=2026-02-16T09:59:19.728381626Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T09:59:19.737816233Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=info ts=2026-02-16T09:59:19.737899114Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=warn ts=2026-02-16T09:59:19.73791228Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=9.533362ms
level=error ts=2026-02-16T09:59:19.737922631Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T09:59:19.737938093Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T09:59:19.743281988Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-02-16T09:59:19.743413241Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T09:59:19.743433318Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T09:59:19.74345017Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T09:59:19.743475492Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T09:59:19.743496187Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T09:59:19.743761276Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-02-16T09:59:19.74377435Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-02-16T09:59:19.743780002Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-02-16T09:59:19.743785274Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T09:59:19.743827218Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
ts=2026-02-16T09:59:19.743858843Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:19.743875805Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=info ts=2026-02-16T09:59:19.743871808Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T09:59:19.743892365Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T09:59:19.744066831Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T09:59:19.744074171Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T09:59:19.744083438Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T09:59:20.745140259Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T09:59:20.938802017Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T09:59:20.938872203Z caller=loki.go:599 msg="Loki started" startup_time=1.277987947s
ts=2026-02-16T09:59:21.692436426Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:21.692461604Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
ts=2026-02-16T09:59:25.328227952Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:25.3282511Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=info ts=2026-02-16T09:59:25.939524328Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T09:59:25.939572351Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T09:59:30.761520224Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-02-16T09:59:30.762419194Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
level=warn ts=2026-02-16T09:59:30.762437264Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="2 errors occurred:\n\t* Failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused\n\t* Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused\n\n"
level=debug ts=2026-02-16T09:59:34.49713569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T09:59:41.987753767Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-02-16T09:59:41.988856882Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-02-16T09:59:41.990163166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.74:7946"
level=info ts=2026-02-16T09:59:41.99078095Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.252836239s
ts=2026-02-16T09:59:50.594226017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39726"
ts=2026-02-16T09:59:51.781282189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39738"
level=debug ts=2026-02-16T09:59:52.468016575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T10:00:14.904945659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T10:00:18.410365983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-12180ce6 10.129.0.76:7946"
ts=2026-02-16T10:00:21.784075793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41452"
level=debug ts=2026-02-16T10:00:39.790751286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T10:00:48.412039252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-9n5vp-f59fd426 10.130.0.75:7946"