level=info ts=2026-02-16T09:59:02.141246524Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)" level=info ts=2026-02-16T09:59:02.141277241Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-02-16T09:59:02.143213458Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-02-16T09:59:02.143693152Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-02-16T09:59:02.1437777Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0 level=debug ts=2026-02-16T09:59:02.143985135Z caller=modules.go:1144 msg="initializing query frontend tripperware" level=error ts=2026-02-16T09:59:02.147472336Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-02-16T09:59:02.147505572Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-02-16T09:59:02.150243145Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-02-16T09:59:02.150263545Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-02-16T09:59:02.151903249Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-02-16T09:59:02.151917054Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-02-16T09:59:02.15623016Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-02-16T09:59:02.156257911Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-02-16T09:59:02.158040661Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-02-16T09:59:02.15805623Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=debug ts=2026-02-16T09:59:02.158408215Z caller=modules.go:1221 msg="initializing query frontend" config="{Handler:{LogQueriesLongerThan:0s LogQueryRequestHeaders: MaxBodySize:10485760 QueryStatsEnabled:false} FrontendV1:{MaxOutstandingPerTenant:2048 QuerierForgetDelay:0s} FrontendV2:{SchedulerAddress:loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095 DNSLookupPeriod:10s WorkerConcurrency:5 GRPCClientConfig:{MaxRecvMsgSize:104857600 MaxSendMsgSize:104857600 GRPCCompression: RateLimit:0 RateLimitBurst:0 BackoffOnRatelimits:false BackoffConfig:{MinBackoff:100ms MaxBackoff:10s MaxRetries:10} InitialStreamWindowSize:65535 InitialConnectionWindowSize:65535 TLSEnabled:false TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:} ConnectTimeout:5s ConnectBackoffBaseDelay:1s ConnectBackoffMaxDelay:5s Middleware:[] StreamMiddleware:[] CustomCompressors:[] ClusterValidation:{Label: registeredFlags:{Prefix:frontend.grpc-client-config.cluster-validation. Flags:map[label:0xc000b197c0]}} clusterUnaryClientInterceptor:} GracefulShutdownTimeout:5m0s InfNames:[eth0 lo] EnableIPv6:false Addr: Port:0 Encoding:json} CompressResponses:false DownstreamURL: TailProxyURL:http://loki-querier.product-kubearchive-logging.svc.cluster.local:3100 TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:} SupportParquetEncoding:false}" level=debug ts=2026-02-16T09:59:02.15842265Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-02-16T09:59:02.158488272Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0 level=debug ts=2026-02-16T09:59:02.15851735Z caller=modules.go:1250 msg="using query frontend" version=v2 level=debug ts=2026-02-16T09:59:02.171357408Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=debug ts=2026-02-16T09:59:02.171368686Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=server level=info ts=2026-02-16T09:59:02.171380267Z caller=module_service.go:82 msg=starting module=server level=info ts=2026-02-16T09:59:02.171386507Z caller=module_service.go:82 msg=starting module=analytics level=debug ts=2026-02-16T09:59:02.171400721Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=runtime-config level=info ts=2026-02-16T09:59:02.17140794Z caller=module_service.go:82 msg=starting module=runtime-config level=info ts=2026-02-16T09:59:02.171423017Z caller=module_service.go:82 msg=starting module=cache-generation-loader level=debug ts=2026-02-16T09:59:02.17143315Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=analytics level=info ts=2026-02-16T09:59:02.171452317Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-02-16T09:59:02.171674238Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=server level=info ts=2026-02-16T09:59:02.171685886Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware level=debug ts=2026-02-16T09:59:02.172035083Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=cache-generation-loader level=debug ts=2026-02-16T09:59:02.172051806Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=memberlist-kv level=debug ts=2026-02-16T09:59:02.172055274Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-frontend-tripperware level=debug ts=2026-02-16T09:59:02.172058495Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=runtime-config level=info ts=2026-02-16T09:59:02.172060801Z caller=module_service.go:82 msg=starting module=query-frontend level=info ts=2026-02-16T09:59:02.172207197Z caller=loki.go:599 msg="Loki started" startup_time=98.590924ms level=warn ts=2026-02-16T09:59:02.175078476Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-02-16T09:59:12.177563669Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=info ts=2026-02-16T09:59:17.904604561Z caller=frontend.go:441 msg="not ready: number of schedulers this worker is connected to is 0" level=debug ts=2026-02-16T09:59:17.904656862Z caller=logging.go:137 msg="GET /ready (503) 102.088µs" level=debug ts=2026-02-16T09:59:19.361074814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=info ts=2026-02-16T09:59:22.179916239Z caller=frontend_scheduler_worker.go:106 msg="adding connection to scheduler" addr=10.130.0.76:9095 level=debug ts=2026-02-16T09:59:42.098179584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-02-16T10:00:02.149201099Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.78:11211 count=1 level=debug ts=2026-02-16T10:00:02.151919297Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.78:11211 count=1 level=debug ts=2026-02-16T10:00:02.153639013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.78:11211 count=1 level=debug ts=2026-02-16T10:00:02.157394997Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.78:11211 count=1 level=debug ts=2026-02-16T10:00:02.159131124Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.78:11211 count=1 level=debug ts=2026-02-16T10:00:04.773414034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-02-16T10:00:33.300927359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"