level=info ts=2026-02-16T13:53:03.917925609Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T13:53:03.917972495Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T13:53:03.919454045Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T13:53:03.921053115Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T13:53:03.921136426Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.73 inf=eth0
level=info ts=2026-02-16T13:53:03.921643389Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-8hh5l-04125aa8
ts=2026-02-16T13:53:03.921701545Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T13:53:03.921731897Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=debug ts=2026-02-16T13:53:03.921757718Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=info ts=2026-02-16T13:53:03.921919008Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T13:53:03.925125936Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T13:53:03.92513904Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:53:03.925146979Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=3.231034ms
level=error ts=2026-02-16T13:53:03.925153292Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T13:53:03.925167232Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T13:53:03.926048976Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T13:53:03.926074645Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-02-16T13:53:03.929600544Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:03.929617062Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:53:03.931901159Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:03.931922541Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T13:53:03.932275962Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T13:53:03.932583853Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T13:53:03.932612585Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-02-16T13:53:03.950113079Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=info ts=2026-02-16T13:53:03.950114155Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T13:53:03.950131603Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T13:53:03.950138999Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T13:53:03.950148768Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-02-16T13:53:03.950168268Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=debug ts=2026-02-16T13:53:03.950179581Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T13:53:03.950186146Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=info ts=2026-02-16T13:53:03.950178352Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T13:53:03.950188337Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T13:53:03.950229719Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T13:53:03.950231062Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:53:03.950241959Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T13:53:03.950249748Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=info ts=2026-02-16T13:53:03.950376068Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T13:53:03.950430434Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T13:53:03.950480131Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T13:53:03.950499216Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T13:53:03.950503631Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-02-16T13:53:03.950515499Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:53:03.950519884Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-02-16T13:53:03.950523286Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-02-16T13:53:03.950520545Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T13:53:03.950541221Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T13:53:03.950545506Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T13:53:03.950548391Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T13:53:03.950551526Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T13:53:03.950526742Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T13:53:03.950561225Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-02-16T13:53:03.950564604Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=info ts=2026-02-16T13:53:03.950567971Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T13:53:03.950626765Z caller=loki.go:599 msg="Loki started" startup_time=98.624373ms
level=warn ts=2026-02-16T13:53:03.953610113Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T13:53:05.792628795Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:05.792657543Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T13:53:06.95651675Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-02-16T13:53:08.380953779Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:08.380977645Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:53:08.928986527Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:08.929013631Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-02-16T13:53:09.958911499Z caller=worker.go:235 component=querier msg="adding connection" addr=10.128.0.70:9095
ts=2026-02-16T13:53:12.983290452Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:12.983317499Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:53:13.930014951Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:13.930040775Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T13:53:18.929580702Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T13:53:18.929612487Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T13:53:22.48796934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:53:24.852368236Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T13:53:24.853438215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
level=info ts=2026-02-16T13:53:24.855019771Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.929845963s
ts=2026-02-16T13:53:35.636308917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T13:53:36.113358195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:53:47.901190634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56426"
level=debug ts=2026-02-16T13:53:57.146865855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:54:03.927366609Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:54:05.638685013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T13:54:08.127013343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49572"
ts=2026-02-16T13:54:16.044364126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45206"
level=debug ts=2026-02-16T13:54:21.274791498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:54:35.641129917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T13:54:46.047419751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:50662"
level=debug ts=2026-02-16T13:54:56.421977508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T13:54:56.423610358Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T13:55:03.927842775Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:55:05.643031416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:55:17.906299193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50652"
ts=2026-02-16T13:55:35.644842957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T13:55:38.133420133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47526"
level=debug ts=2026-02-16T13:55:39.746088955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:55:47.909225457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59100"
level=debug ts=2026-02-16T13:56:03.927902732Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:56:05.646447627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:56:16.055847928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46106"
level=debug ts=2026-02-16T13:56:29.984773012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T13:56:35.648594499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T13:56:47.915090553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55894"
level=debug ts=2026-02-16T13:57:03.927499063Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:57:05.650550421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T13:57:17.917277284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49358"
ts=2026-02-16T13:57:35.653180678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T13:57:38.14247056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:41316"
level=debug ts=2026-02-16T13:57:45.892637472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:58:03.928149297Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:58:05.65525156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:58:35.656286355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T13:58:46.06978589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56616"
ts=2026-02-16T13:58:47.924883769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33116"
level=debug ts=2026-02-16T13:58:58.450651718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T13:59:03.927833034Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T13:59:05.657958261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T13:59:17.926931405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46674"
ts=2026-02-16T13:59:35.660568727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:00:03.927562188Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:00:05.662145463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:00:10.304977126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:00:10.30683875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:00:16.079109987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:51090"
ts=2026-02-16T14:00:35.663497278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:00:46.083313415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:58226"
ts=2026-02-16T14:00:47.933879245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55520"
level=debug ts=2026-02-16T14:01:02.686861143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:01:03.928118241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:01:05.665451964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:01:08.160165722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46360"
ts=2026-02-16T14:01:35.667498595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:01:38.161583368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43464"
level=debug ts=2026-02-16T14:01:54.915061924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:02:03.92749257Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:02:05.670700502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:02:35.672669652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:02:38.165829587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34902"
level=debug ts=2026-02-16T14:02:52.816568196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:03:03.928043656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:03:05.673704832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:03:35.676333348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:03:46.101835594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41790"
ts=2026-02-16T14:03:47.950348638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36336"
level=debug ts=2026-02-16T14:03:48.620487598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:04:03.92765515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:04:05.67877977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:04:31.88238633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:04:31.884161046Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:04:35.681191334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:05:03.927719707Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:05:05.683687272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:05:16.111593408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35470"
ts=2026-02-16T14:05:17.959809731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36450"
level=debug ts=2026-02-16T14:05:32.471053476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:05:35.68904054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:05:46.114400209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45846"
level=debug ts=2026-02-16T14:06:03.927489862Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:06:05.691113701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:06:08.183989782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35282"
level=debug ts=2026-02-16T14:06:09.454396116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:06:35.693116643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:06:46.120361519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37968"
level=debug ts=2026-02-16T14:07:02.752032843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:07:03.927842521Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:07:05.69495651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:07:17.970862613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44776"
ts=2026-02-16T14:07:35.697007373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:07:38.191076298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52836"
level=debug ts=2026-02-16T14:07:51.816746829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:08:03.92761972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:08:05.698751573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:08:17.9753384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54082"
ts=2026-02-16T14:08:35.699641019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:08:53.939444193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:08:53.942246988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:09:03.930086368Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:09:05.700996365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:09:16.143060713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40420"
level=debug ts=2026-02-16T14:09:30.390129768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:09:35.702094536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:09:46.145527632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36558"
ts=2026-02-16T14:09:47.983055868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35332"
level=debug ts=2026-02-16T14:10:03.927729874Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:10:05.703793763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:10:17.985381175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35436"
level=debug ts=2026-02-16T14:10:20.54213424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:10:35.706202141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:10:38.207464952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35684"
level=debug ts=2026-02-16T14:11:03.927625277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:11:05.714365802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
level=debug ts=2026-02-16T14:11:22.075927342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:11:35.737423025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:11:38.220385249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43126"
level=debug ts=2026-02-16T14:12:03.92812056Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:12:05.740016315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:12:08.22460323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58688"
level=debug ts=2026-02-16T14:12:09.693071835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:12:16.160015317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41044"
ts=2026-02-16T14:12:35.742348408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:12:46.16286232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41428"
level=debug ts=2026-02-16T14:12:57.660113779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:12:57.661746407Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:13:03.926959306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:13:05.744330191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:13:08.228251458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55404"
ts=2026-02-16T14:13:35.745824136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:13:41.039387095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:14:03.931270348Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:14:05.747827518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:14:35.750621007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:14:38.234677517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46806"
level=debug ts=2026-02-16T14:14:40.439673223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:15:03.934406935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:15:05.754896589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-daad38b4 10.130.0.68:7946"
ts=2026-02-16T14:15:16.179535562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60034"
level=debug ts=2026-02-16T14:15:19.049476402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:15:35.756958484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:15:38.239922951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50724"
ts=2026-02-16T14:15:46.182641366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38618"
level=debug ts=2026-02-16T14:16:03.541045007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:16:03.943685766Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:16:05.759470613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:16:15.445306748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41796"
ts=2026-02-16T14:16:15.456616177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41806"
ts=2026-02-16T14:16:35.762591013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:16:38.245709072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35936"
level=debug ts=2026-02-16T14:16:45.135998623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:16:45.137869894Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:16:45.925275222Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-daad38b4 (timeout reached)"
ts=2026-02-16T14:16:48.924269575Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-daad38b4 has failed, no acks received"
ts=2026-02-16T14:16:56.98761482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53196"
level=debug ts=2026-02-16T14:17:03.928295972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:17:05.764655109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:17:24.022316358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:17:35.766952971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:17:46.19432706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40164"
level=debug ts=2026-02-16T14:18:03.92788249Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:18:05.769385156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
level=debug ts=2026-02-16T14:18:25.540352627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:18:26.994579704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43608"
ts=2026-02-16T14:18:35.77173517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:18:38.255818054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40892"
level=debug ts=2026-02-16T14:19:03.928132136Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:19:05.7740822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:19:23.757486761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:19:35.77628236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:19:46.207830999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:48064"
level=debug ts=2026-02-16T14:19:58.741509461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:20:03.928059163Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:20:05.778300899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:20:08.263256448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52294"
ts=2026-02-16T14:20:27.004926723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49214"
ts=2026-02-16T14:20:35.780317239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:20:53.536142099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:20:53.537897095Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:21:03.927765251Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:21:05.78273838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:21:08.268171691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43074"
ts=2026-02-16T14:21:16.217054944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43734"
ts=2026-02-16T14:21:35.784438841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
level=debug ts=2026-02-16T14:21:39.523254716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:21:46.219993915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47946"
level=debug ts=2026-02-16T14:22:03.929485204Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:22:05.786403313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:22:25.302564075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:22:35.78829843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a0c3718 10.130.0.68:7946"
ts=2026-02-16T14:22:46.225958961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49704"
ts=2026-02-16T14:22:57.016745553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59206"
level=debug ts=2026-02-16T14:23:03.927855397Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:23:05.790488909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:23:06.099128471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:23:08.278239278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53840"
ts=2026-02-16T14:23:16.228902998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60684"
ts=2026-02-16T14:23:35.792146399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:23:38.280114161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59646"
level=debug ts=2026-02-16T14:23:54.103288279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:23:57.0222619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43774"
level=debug ts=2026-02-16T14:24:03.927501192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:24:05.794637791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:24:08.281298329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33132"
ts=2026-02-16T14:24:25.923821897Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1a0c3718 (timeout reached)"
ts=2026-02-16T14:24:28.923088811Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1a0c3718 has failed, no acks received"
ts=2026-02-16T14:24:35.796623189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:24:38.28323207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49228"
ts=2026-02-16T14:24:38.370546832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41536"
ts=2026-02-16T14:24:38.382118953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41548"
ts=2026-02-16T14:24:45.924314028Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1a0c3718 (timeout reached)"
ts=2026-02-16T14:24:48.922652993Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1a0c3718 has failed, no acks received"
ts=2026-02-16T14:24:48.923176768Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1a0c3718 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-02-16T14:24:54.701578215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:24:54.703267248Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:25:03.928017728Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:25:05.798641971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:25:16.241410116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55756"
ts=2026-02-16T14:25:35.800574304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:25:37.903059195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:25:38.287145566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37962"
ts=2026-02-16T14:25:58.196771051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56782"
level=debug ts=2026-02-16T14:26:03.927577584Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:26:05.802862652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
level=debug ts=2026-02-16T14:26:13.96568525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:26:28.199250197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47296"
ts=2026-02-16T14:26:35.805130535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:26:58.201684973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47382"
level=debug ts=2026-02-16T14:27:03.928043295Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=debug ts=2026-02-16T14:27:05.073760418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:27:05.807704448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:27:08.293328118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46752"
ts=2026-02-16T14:27:16.254643638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:51558"
ts=2026-02-16T14:27:35.809973542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:27:58.206107338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41168"
level=debug ts=2026-02-16T14:28:01.89052523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:28:03.927982682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:28:05.81212362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:28:08.296807179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52872"
ts=2026-02-16T14:28:35.814622252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:28:46.26333153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35616"
level=debug ts=2026-02-16T14:28:55.94966257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:28:55.951165394Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T14:29:03.927837711Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:29:05.816749048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:29:08.299930537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53370"
ts=2026-02-16T14:29:16.266535826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52206"
ts=2026-02-16T14:29:35.818938351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:29:45.077181431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:29:46.269288782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:48268"
ts=2026-02-16T14:29:58.217750887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57938"
level=debug ts=2026-02-16T14:30:03.927745014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:30:05.820725049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:30:28.220080769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45336"
level=debug ts=2026-02-16T14:30:29.608790231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:30:35.823189706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ee30c144 10.130.0.68:7946"
ts=2026-02-16T14:30:46.275339978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53966"
level=debug ts=2026-02-16T14:31:03.927688697Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:31:05.826430419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:31:21.858614072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:31:28.225479688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36496"
ts=2026-02-16T14:31:35.827988575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:31:38.312640394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57818"
ts=2026-02-16T14:31:46.280899977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43308"
level=debug ts=2026-02-16T14:32:03.928183542Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:32:05.829620509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:32:16.283422059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60790"
level=debug ts=2026-02-16T14:32:23.816053589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:32:35.832059493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:32:38.316493941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52264"
ts=2026-02-16T14:32:58.234267857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39396"
level=debug ts=2026-02-16T14:33:03.927343463Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:33:05.834042654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:33:10.92352203Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee30c144 (timeout reached)"
ts=2026-02-16T14:33:13.923250916Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee30c144 has failed, no acks received"
level=debug ts=2026-02-16T14:33:16.62024739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T14:33:16.621791338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T14:33:25.922777534Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee30c144 (timeout reached)"
ts=2026-02-16T14:33:28.922508897Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee30c144 has failed, no acks received"
ts=2026-02-16T14:33:33.332457212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33146"
ts=2026-02-16T14:33:33.344104632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33162"
ts=2026-02-16T14:33:35.836693097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:33:35.923312509Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ee30c144 (timeout reached)"
ts=2026-02-16T14:33:38.319919444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43592"
ts=2026-02-16T14:33:38.923231134Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ee30c144 has failed, no acks received"
ts=2026-02-16T14:33:46.291031578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49744"
level=debug ts=2026-02-16T14:34:03.927369381Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:34:05.839855999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:34:08.381963773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:34:35.842802466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-81e5c3b7 10.130.0.68:7946"
ts=2026-02-16T14:34:38.32288567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58236"
ts=2026-02-16T14:34:40.595086286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48630"
level=debug ts=2026-02-16T14:34:50.733781271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:35:03.927126977Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:35:05.844470826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
ts=2026-02-16T14:35:10.597339324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33828"
level=debug ts=2026-02-16T14:35:33.239344195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:35:35.846583608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-81e5c3b7 10.130.0.68:7946"
level=debug ts=2026-02-16T14:36:03.927951175Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
ts=2026-02-16T14:36:05.848547278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=debug ts=2026-02-16T14:36:28.322703322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T14:36:35.850721071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
ts=2026-02-16T14:36:38.330762683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60108"