level=info ts=2026-05-06T20:51:07.084925388Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)" level=info ts=2026-05-06T20:51:07.084954727Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-05-06T20:51:07.087576562Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-05-06T20:51:07.08808116Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-06T20:51:07.088187791Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.52 inf=eth0 level=debug ts=2026-05-06T20:51:07.088460396Z caller=modules.go:1164 msg="initializing query frontend tripperware" level=error ts=2026-05-06T20:51:07.092582077Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:51:07.092614714Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-05-06T20:51:07.095544535Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:51:07.095575751Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-05-06T20:51:07.097360704Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:51:07.097375773Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-05-06T20:51:07.102273385Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:51:07.102291441Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=error ts=2026-05-06T20:51:07.103937931Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-results-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:51:07.103955728Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0 level=debug ts=2026-05-06T20:51:07.104292115Z caller=modules.go:1268 msg="initializing query frontend" config="{Handler:{LogQueriesLongerThan:0s LogQueryRequestHeaders: MaxBodySize:10485760 QueryStatsEnabled:false} FrontendV1:{MaxOutstandingPerTenant:2048 QuerierForgetDelay:0s} FrontendV2:{SchedulerAddress:loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095 DNSLookupPeriod:10s WorkerConcurrency:5 GRPCClientConfig:{MaxRecvMsgSize:104857600 MaxSendMsgSize:104857600 GRPCCompression: RateLimit:0 RateLimitBurst:0 BackoffOnRatelimits:false BackoffConfig:{MinBackoff:100ms MaxBackoff:10s MaxRetries:10} InitialStreamWindowSize:65535 InitialConnectionWindowSize:65535 TLSEnabled:false TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:} ConnectTimeout:5s ConnectBackoffBaseDelay:1s ConnectBackoffMaxDelay:5s Middleware:[] StreamMiddleware:[] CustomCompressors:[] ClusterValidation:{Label: registeredFlags:{Prefix:frontend.grpc-client-config.cluster-validation. Flags:map[label:0x13500ebd6240]}} clusterUnaryClientInterceptor:} GracefulShutdownTimeout:5m0s InfNames:[eth0 lo] EnableIPv6:false Addr: Port:0 Encoding:json} CompressResponses:false DownstreamURL: TailProxyURL:http://loki-querier.product-kubearchive-logging.svc.cluster.local:3100 TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:} SupportParquetEncoding:false}" level=debug ts=2026-05-06T20:51:07.104304805Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-06T20:51:07.10437165Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.52 inf=eth0 level=debug ts=2026-05-06T20:51:07.1044018Z caller=modules.go:1297 msg="using query frontend" version=v2 level=debug ts=2026-05-06T20:51:07.120884756Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=info ts=2026-05-06T20:51:07.120914121Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-05-06T20:51:07.120903174Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=runtime-config level=debug ts=2026-05-06T20:51:07.120941865Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=server level=info ts=2026-05-06T20:51:07.120955596Z caller=module_service.go:82 msg=starting module=cache-generation-loader level=info ts=2026-05-06T20:51:07.120968548Z caller=module_service.go:82 msg=starting module=analytics level=info ts=2026-05-06T20:51:07.120980301Z caller=module_service.go:82 msg=starting module=server level=debug ts=2026-05-06T20:51:07.121052337Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=analytics level=info ts=2026-05-06T20:51:07.121077946Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-05-06T20:51:07.121158787Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=cache-generation-loader level=debug ts=2026-05-06T20:51:07.121170048Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=memberlist-kv level=debug ts=2026-05-06T20:51:07.121175346Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-frontend-tripperware level=debug ts=2026-05-06T20:51:07.12120611Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=server level=info ts=2026-05-06T20:51:07.121211739Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware level=debug ts=2026-05-06T20:51:07.121222812Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=runtime-config level=info ts=2026-05-06T20:51:07.121227692Z caller=module_service.go:82 msg=starting module=query-frontend level=info ts=2026-05-06T20:51:07.121326339Z caller=loki.go:615 msg="Loki started" startup_time=117.271746ms level=info ts=2026-05-06T20:51:07.125028485Z caller=frontend_scheduler_worker.go:106 msg="adding connection to scheduler" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.126010558Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.126041522Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.126056822Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.126289366Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.126306992Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.725553996Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.753071809Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.790392789Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.83848765Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=error ts=2026-05-06T20:51:07.893788906Z caller=frontend_scheduler_worker.go:237 msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.108.30:9095: connect: connection refused\"" addr=172.30.108.30:9095 level=debug ts=2026-05-06T20:51:22.590215415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:51:41.748718887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:51:55.808973344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:52:07.094400311Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:52:07.097263566Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:52:07.099045965Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:52:07.103886781Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:52:07.105653173Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:52:19.010484456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:52:44.254992442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-06T20:52:44.256578033Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:53:07.094765569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:53:07.096771604Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:53:07.098582018Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:53:07.103516079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:53:07.105414737Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:53:30.177300281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:54:07.093774131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:54:07.096697944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:54:07.098549768Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:54:07.103323646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:54:07.10498597Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:54:47.709978307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:55:07.094633592Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:55:07.09647026Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:55:07.098219664Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:55:07.103071695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:55:07.104815417Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:55:44.670037955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:56:07.094419118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:56:07.09725778Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:56:07.099097328Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:56:07.103951921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:56:07.105680153Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:56:53.354931998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=debug ts=2026-05-06T20:57:07.094667603Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:57:07.096362316Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:57:07.098018937Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:57:07.103986085Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:57:07.104602859Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:05.705139189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-06T20:58:05.707062232Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T20:58:07.094244883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:07.09707263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:07.098853515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:07.103818342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:07.105569955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:58:50.097159951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T20:59:07.094259397Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:59:07.097281794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:59:07.09898789Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:59:07.10380036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:59:07.105393459Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T20:59:43.431584633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:00:07.094223331Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:00:07.096958848Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:00:07.09868785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:00:07.103885242Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:00:07.105602151Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:00:26.518334519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:01:05.036238692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:01:07.094088982Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:01:07.096790089Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:01:07.098385361Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:01:07.10314685Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:01:07.104687722Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:02:05.738382566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-06T21:02:05.739996108Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-06T21:02:07.093737351Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:02:07.096487656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:02:07.098217278Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:02:07.102958545Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:02:07.104697649Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:02.708565697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:03:07.093979649Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:07.096707123Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:07.098332735Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:07.103000968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:07.104613661Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:03:39.727763511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:04:07.09371961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:04:07.096578515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:04:07.098382355Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:04:07.103213209Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:04:07.104925001Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:04:37.422627041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:05:07.094466475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:05:07.097350847Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:05:07.099044403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:05:07.104112004Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:05:07.104637827Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:05:34.378885894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=debug ts=2026-05-06T21:06:07.094042693Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:06:07.096936162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:06:07.098766894Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:06:07.103514562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1 level=debug ts=2026-05-06T21:06:07.105212145Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.54:11211 count=1