level=info ts=2026-02-16T11:31:01.009761255Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T11:31:01.009827397Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T11:31:01.010719608Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T11:31:01.012220328Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:31:01.012297946Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.58 inf=eth0
level=info ts=2026-02-16T11:31:01.01316264Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-77cddbcfd9-xzbjr-db1d6c30
ts=2026-02-16T11:31:01.013230927Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T11:31:01.013243631Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=debug ts=2026-02-16T11:31:01.013264125Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=info ts=2026-02-16T11:31:01.013366692Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-02-16T11:31:01.017444Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:31:01.017484457Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-02-16T11:31:01.020573351Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:31:01.020600825Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:31:01.020626762Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.262193ms
level=error ts=2026-02-16T11:31:01.020637836Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T11:31:01.020652124Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:01.022903793Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:01.022927235Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:01.023080894Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:01.023115004Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T11:31:01.023584826Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-02-16T11:31:01.02390948Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-02-16T11:31:01.023954118Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-02-16T11:31:01.04000943Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T11:31:01.040025582Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:31:01.04005322Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:31:01.040053885Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T11:31:01.040066853Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T11:31:01.040069493Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T11:31:01.040067207Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=info ts=2026-02-16T11:31:01.040084353Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-02-16T11:31:01.040105727Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T11:31:01.04027432Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T11:31:01.040294445Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T11:31:01.040302893Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-02-16T11:31:01.040335578Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-02-16T11:31:01.040436286Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-02-16T11:31:01.040430983Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T11:31:01.040448742Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=info ts=2026-02-16T11:31:01.040449714Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-02-16T11:31:01.040480838Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-02-16T11:31:01.04050804Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T11:31:01.040511992Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-02-16T11:31:01.040514986Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=debug ts=2026-02-16T11:31:01.040518826Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-02-16T11:31:01.04053633Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-02-16T11:31:01.04054436Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2026-02-16T11:31:01.040551794Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T11:31:01.040557778Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-02-16T11:31:01.040560809Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-02-16T11:31:01.04056503Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-02-16T11:31:01.040567732Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:31:01.040570384Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=info ts=2026-02-16T11:31:01.040572679Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-02-16T11:31:01.040653045Z caller=loki.go:599 msg="Loki started" startup_time=91.687372ms
level=info ts=2026-02-16T11:31:01.044246121Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.53:9095
ts=2026-02-16T11:31:02.639165042Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:02.63919609Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:06.022512631Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:06.022565097Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:06.113713706Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:06.113744509Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:10.260627994Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:10.260652159Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:11.021492276Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:11.02151941Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-02-16T11:31:16.021488583Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:16.021544229Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T11:31:17.479136457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:19.345494879Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:19.345536972Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:31:21.021426195Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:21.021453975Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T11:31:37.413143103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:39.021948118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33068"
ts=2026-02-16T11:31:45.011558676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:46884"
ts=2026-02-16T11:31:46.565274336Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.60:7946: dial tcp 10.129.0.60:7946: connect: connection refused"
ts=2026-02-16T11:31:46.566276935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.61:7946"
ts=2026-02-16T11:31:46.568946353Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.63:7946: dial tcp 10.129.0.63:7946: connect: connection refused"
ts=2026-02-16T11:31:46.569943237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.54:7946"
ts=2026-02-16T11:31:46.572271838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.55:7946"
ts=2026-02-16T11:31:46.574702827Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.53:7946: dial tcp 10.130.0.53:7946: connect: connection refused"
ts=2026-02-16T11:31:46.574797474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.58:7946"
ts=2026-02-16T11:31:46.574809325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40616"
level=info ts=2026-02-16T11:31:46.575129183Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=45.554471019s
ts=2026-02-16T11:31:47.808427215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:31:51.124639147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:49652"
level=debug ts=2026-02-16T11:32:00.292807875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:32:01.019284678Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:32:06.390983251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56224"
ts=2026-02-16T11:32:17.811877278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:32:23.189173278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:32:29.827511145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:52292"
ts=2026-02-16T11:32:36.394481989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:35252"
ts=2026-02-16T11:32:47.814853871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:32:59.830657465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39468"
level=debug ts=2026-02-16T11:33:00.420810705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:33:00.42267168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:33:01.018867377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:33:17.818338713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:33:29.833672324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:34030"
level=debug ts=2026-02-16T11:33:36.656016943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:33:47.821672309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:34:01.019558546Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:34:06.402511736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56900"
ts=2026-02-16T11:34:17.824747492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:34:29.839499348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45576"
ts=2026-02-16T11:34:47.828233181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:34:54.713660185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:35:01.018564139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:35:17.830779321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:35:29.844002276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37906"
ts=2026-02-16T11:35:47.833613305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:36:01.018963091Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:36:08.662189891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:36:11.184708972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:51472"
ts=2026-02-16T11:36:17.836332473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:36:29.849856686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:39990"
ts=2026-02-16T11:36:36.41235631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:54926"
ts=2026-02-16T11:36:41.188224611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60568"
ts=2026-02-16T11:36:47.839584148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:37:01.019512298Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:37:03.184581587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:37:11.191744719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60754"
ts=2026-02-16T11:37:17.8423427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:37:36.415969099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50624"
ts=2026-02-16T11:37:47.845726842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:37:55.814177312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:37:55.815894111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:37:59.85796207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:33348"
level=debug ts=2026-02-16T11:38:01.019158561Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:38:06.419405391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47264"
ts=2026-02-16T11:38:17.848437323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:38:36.422928893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58644"
ts=2026-02-16T11:38:41.19873291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:36122"
ts=2026-02-16T11:38:47.851881239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:38:52.488195885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:39:01.019115911Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:39:17.854727897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:39:29.926355984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:39:41.202512244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:41354"
ts=2026-02-16T11:39:47.85826446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:40:01.019263068Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:40:17.861922849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:40:23.439830311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:40:29.868878542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54442"
ts=2026-02-16T11:40:47.864927084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:41:01.018942604Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:41:06.433325067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45104"
ts=2026-02-16T11:41:17.867597322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:41:24.188847691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:41:47.870490334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:42:01.019052819Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:42:17.873023014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:42:26.101638216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:42:26.103602553Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:42:41.216671387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:43648"
ts=2026-02-16T11:42:47.87634333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:43:01.019049882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:43:06.442740966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41116"
level=debug ts=2026-02-16T11:43:13.146885484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:43:17.879461985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:43:29.88374555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:51116"
ts=2026-02-16T11:43:47.882537311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:44:01.019583003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:44:01.710650429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:44:17.886613796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T11:44:41.226895311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:37970"
ts=2026-02-16T11:44:47.890305836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:44:53.464905673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:45:01.018562907Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:45:11.233509197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:41362"
ts=2026-02-16T11:45:17.893202508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T11:45:36.456402688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37736"
level=debug ts=2026-02-16T11:45:44.213367592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:45:47.8960305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:45:59.897396207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:48912"
level=debug ts=2026-02-16T11:46:01.019306226Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:46:06.459556835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41658"
ts=2026-02-16T11:46:17.898784666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:46:28.407004724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:46:28.409177805Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:46:29.900421928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:33764"
ts=2026-02-16T11:46:41.243411299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:50336"
ts=2026-02-16T11:46:47.902127138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:47:01.019473876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:47:09.473174738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:47:17.905424077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:47:41.250271599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:34178"
ts=2026-02-16T11:47:47.908551782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:47:59.908221687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45314"
level=debug ts=2026-02-16T11:48:01.020375688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:48:01.222742748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:48:17.911340145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:48:36.472036949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56412"
ts=2026-02-16T11:48:47.914905778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:48:48.917131041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:49:01.019598949Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:49:17.918058355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:49:44.375251595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:49:47.921652771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:50:01.018980552Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:50:11.269833229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:45966"
ts=2026-02-16T11:50:17.925012157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:50:26.976730775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:50:26.978556273Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:50:47.92909416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:50:59.925980511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:54350"
level=debug ts=2026-02-16T11:51:01.019778817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:51:06.48323474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:41792"
level=debug ts=2026-02-16T11:51:09.346993943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:51:17.931824116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
ts=2026-02-16T11:51:36.486370601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45220"
ts=2026-02-16T11:51:41.276850802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:40418"
ts=2026-02-16T11:51:47.934553864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:51:57.773044745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T11:52:01.018512232Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:52:06.489816757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:39832"
ts=2026-02-16T11:52:17.938316986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:52:36.493386927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60636"
ts=2026-02-16T11:52:47.941595335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:52:59.570052791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:52:59.941226044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45670"
level=debug ts=2026-02-16T11:53:01.018976323Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:53:06.496595179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:34228"
ts=2026-02-16T11:53:17.944554886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:53:29.944420495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:47948"
level=debug ts=2026-02-16T11:53:41.158514497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:53:41.284443889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:52858"
ts=2026-02-16T11:53:47.947875201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:54:01.019304015Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:54:17.950968606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:54:28.77293861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:54:28.77592615Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:54:41.289788104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:56868"
ts=2026-02-16T11:54:47.95383738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:55:01.019123006Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:55:05.207680952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:55:17.95683005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:55:29.954565878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:51210"
ts=2026-02-16T11:55:47.96048887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:55:50.94183041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:55:59.957657347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:59852"
level=debug ts=2026-02-16T11:56:01.028253382Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:56:17.963260604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f64719b 10.130.0.55:7946"
level=debug ts=2026-02-16T11:56:31.64195018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:56:47.966652451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:57:01.033052031Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:57:01.83420087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:44968"
ts=2026-02-16T11:57:01.843864343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:44980"
ts=2026-02-16T11:57:17.970243025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:57:18.602444595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:57:28.01815754Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3f64719b (timeout reached)"
ts=2026-02-16T11:57:31.015626778Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3f64719b has failed, no acks received"
ts=2026-02-16T11:57:47.973784829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T11:58:01.019063451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T11:58:06.759704536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:58:06.76174044Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:58:17.976666482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
ts=2026-02-16T11:58:47.979515729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T11:58:56.15667507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:58:59.975381604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:51262"
level=debug ts=2026-02-16T11:59:01.019171071Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T11:59:05.435660287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:50436"
ts=2026-02-16T11:59:17.982800168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
ts=2026-02-16T11:59:29.984854686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:44290"
ts=2026-02-16T11:59:35.438351846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:54242"
level=debug ts=2026-02-16T11:59:42.584862984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:59:47.986283788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T11:59:59.989361287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50962"
level=debug ts=2026-02-16T12:00:01.01867212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:00:17.989186099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:00:29.992397722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:55812"
ts=2026-02-16T12:00:35.443437566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:48778"
level=debug ts=2026-02-16T12:00:35.64694658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:00:36.535939725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:58484"
ts=2026-02-16T12:00:47.992082288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:01:01.018839389Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:01:05.447097529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:48822"
level=debug ts=2026-02-16T12:01:12.880658951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:01:17.994697537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:01:47.722329088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:01:47.725191401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:01:47.997827007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:02:01.018950083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:02:18.000686233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:02:25.435271533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:02:30.003281099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37130"
ts=2026-02-16T12:02:36.546173299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56684"
ts=2026-02-16T12:02:48.003219424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:03:01.02044618Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T12:03:04.156932909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:03:06.549184208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:56664"
ts=2026-02-16T12:03:18.006440805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:03:35.458015131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:49080"
ts=2026-02-16T12:03:48.009041455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:03:56.289911884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:00.011686192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:57880"
level=debug ts=2026-02-16T12:04:01.01881286Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:04:18.011762113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:04:30.014613746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50450"
level=debug ts=2026-02-16T12:04:36.341507697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:48.015424184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:05:01.019429597Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:05:06.557636168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:52370"
ts=2026-02-16T12:05:18.018644284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3c0b0408 10.130.0.55:7946"
level=debug ts=2026-02-16T12:05:18.169978766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:05:18.171786438Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:05:30.020723492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:60038"
ts=2026-02-16T12:05:36.560780724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40074"
ts=2026-02-16T12:05:43.014972984Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3c0b0408 (timeout reached)"
ts=2026-02-16T12:05:46.014106479Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3c0b0408 has failed, no acks received"
ts=2026-02-16T12:05:48.014864712Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3c0b0408 (timeout reached)"
ts=2026-02-16T12:05:48.022162241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:05:51.014895887Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3c0b0408 has failed, no acks received"
ts=2026-02-16T12:05:54.17357029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:46882"
ts=2026-02-16T12:05:54.182406911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:46886"
ts=2026-02-16T12:06:00.024308735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:35000"
level=debug ts=2026-02-16T12:06:01.018835921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T12:06:05.085009806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:06:06.014701309Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3c0b0408 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:06:08.014141085Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3c0b0408 (timeout reached)"
ts=2026-02-16T12:06:11.01408158Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3c0b0408 has failed, no acks received"
ts=2026-02-16T12:06:18.02543615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:06:48.02879017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:06:59.895673085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:07:00.035057436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37738"
level=debug ts=2026-02-16T12:07:01.018474927Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:07:06.567711764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33212"
ts=2026-02-16T12:07:18.033491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:07:48.036705175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
level=debug ts=2026-02-16T12:07:51.768614272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:08:01.018740634Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:08:18.039724821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
ts=2026-02-16T12:08:30.047226731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:50708"
level=debug ts=2026-02-16T12:08:41.045819872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:08:48.042911328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:09:01.018687321Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:09:06.576429773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:43450"
ts=2026-02-16T12:09:16.813441726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:42062"
ts=2026-02-16T12:09:18.04572839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:09:41.098934956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:09:41.100628302Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:09:48.04937747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:10:01.023457132Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:10:16.819965814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:46138"
level=debug ts=2026-02-16T12:10:17.025090753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:10:18.052205309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:10:46.822842562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:44652"
ts=2026-02-16T12:10:48.055351541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
level=debug ts=2026-02-16T12:10:51.377879286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:11:01.018897858Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:11:06.587984275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:60614"
ts=2026-02-16T12:11:18.058352728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:11:30.061828129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:33464"
ts=2026-02-16T12:11:46.831043218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:45416"
ts=2026-02-16T12:11:48.061565327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:11:50.003683662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:12:01.01929476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:12:18.064353721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b4cd34c9 10.130.0.55:7946"
ts=2026-02-16T12:12:30.067266413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:59778"
ts=2026-02-16T12:12:36.595513887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45050"
ts=2026-02-16T12:12:48.067876243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:12:49.435117095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:13:01.018951107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:13:16.837664817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47852"
ts=2026-02-16T12:13:18.07093369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:13:25.355108387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:13:25.356760666Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:13:36.600151733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50452"
ts=2026-02-16T12:13:48.074131788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:14:00.074456524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:47534"
level=debug ts=2026-02-16T12:14:01.01882284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:14:08.014722793Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b4cd34c9 (timeout reached)"
ts=2026-02-16T12:14:11.015252928Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b4cd34c9 has failed, no acks received"
ts=2026-02-16T12:14:13.016001858Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b4cd34c9 (timeout reached)"
ts=2026-02-16T12:14:16.016254628Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b4cd34c9 has failed, no acks received"
level=debug ts=2026-02-16T12:14:16.939624352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:14:18.077207166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:14:28.014508388Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b4cd34c9 (timeout reached)"
ts=2026-02-16T12:14:31.014590414Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b4cd34c9 has failed, no acks received"
ts=2026-02-16T12:14:31.015674957Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b4cd34c9 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:14:35.153331095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:38494"
ts=2026-02-16T12:14:35.16291988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:38510"
ts=2026-02-16T12:14:36.605755954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:47452"
ts=2026-02-16T12:14:48.080023258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:15:00.852494134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:15:01.019353609Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:15:06.608412695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46058"
ts=2026-02-16T12:15:18.082694145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:15:30.082234368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:40328"
ts=2026-02-16T12:15:36.611409148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:37280"
level=debug ts=2026-02-16T12:15:42.26398988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:15:48.086367254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:16:01.019277322Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:16:06.614401377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:40188"
ts=2026-02-16T12:16:18.089666046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:16:30.087715598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:52902"
ts=2026-02-16T12:16:36.617930984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:36386"
level=debug ts=2026-02-16T12:16:37.353687971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:16:48.09304714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:16:56.5158752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:56282"
level=debug ts=2026-02-16T12:17:01.020025355Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:17:06.621148454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46734"
ts=2026-02-16T12:17:18.09565261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:17:22.500288123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:17:22.502318137Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:17:48.09841532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:17:56.519796442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47738"
level=debug ts=2026-02-16T12:17:59.22970626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:18:01.018794777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:18:18.102176745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:18:30.09696251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:45382"
ts=2026-02-16T12:18:36.627425056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:50770"
ts=2026-02-16T12:18:48.105190575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:18:48.148759196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:18:56.524378239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:42464"
ts=2026-02-16T12:19:00.100277383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:49818"
level=debug ts=2026-02-16T12:19:01.019928561Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:19:18.108537978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:19:41.420391052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:19:48.112062596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:20:01.019113749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:20:06.635470177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:53056"
ts=2026-02-16T12:20:18.115717041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:20:23.427132706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:20:48.118587584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:20:56.533038406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:34094"
level=debug ts=2026-02-16T12:21:01.019820929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T12:21:12.822680973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:21:12.82525751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:21:18.121808303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:21:30.113048565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:52196"
ts=2026-02-16T12:21:48.125076248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:21:57.658950528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:22:01.018766279Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:22:18.128353645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
ts=2026-02-16T12:22:30.118380072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:57864"
ts=2026-02-16T12:22:48.132082932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-929db2e7 10.130.0.55:7946"
level=debug ts=2026-02-16T12:22:55.926329634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:22:56.540177268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:52164"
level=debug ts=2026-02-16T12:23:01.019089997Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:23:18.135427654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:23:30.947760389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:23:36.651103022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:33736"
ts=2026-02-16T12:23:48.138553483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:24:01.019304967Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:24:03.014033603Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-929db2e7 (timeout reached)"
ts=2026-02-16T12:24:06.014707025Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-929db2e7 has failed, no acks received"
ts=2026-02-16T12:24:06.654042988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:49282"
level=debug ts=2026-02-16T12:24:13.905090156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:24:18.141275284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
ts=2026-02-16T12:24:26.015646769Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-929db2e7 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-02-16T12:24:28.01459298Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-929db2e7 (timeout reached)"
ts=2026-02-16T12:24:31.014273832Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-929db2e7 has failed, no acks received"
ts=2026-02-16T12:24:43.142896203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47372"
ts=2026-02-16T12:24:43.151780866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:47386"
ts=2026-02-16T12:24:48.144033523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-dj97m-6d7423c0 10.129.0.61:7946"
level=debug ts=2026-02-16T12:24:57.649204391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:24:57.650988714Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:25:01.01988656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:25:06.659075257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57002"
ts=2026-02-16T12:25:18.147257131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:25:36.661933752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:57104"
level=debug ts=2026-02-16T12:25:36.691170708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:25:48.150751643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:26:01.019342859Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:26:18.154012509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
level=debug ts=2026-02-16T12:26:18.415519944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:26:48.156998503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:27:01.01937162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:27:06.669478109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:55910"
level=debug ts=2026-02-16T12:27:07.840565083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:27:18.160363089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-08edc817 10.130.0.54:7946"
ts=2026-02-16T12:27:30.143477534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:37348"
ts=2026-02-16T12:27:36.672568916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:46722"
ts=2026-02-16T12:27:48.163189618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
ts=2026-02-16T12:28:00.146124339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:52176"
level=debug ts=2026-02-16T12:28:01.019350606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
level=debug ts=2026-02-16T12:28:03.880389495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:28:18.166679312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
ts=2026-02-16T12:28:30.149060426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.61:49632"
ts=2026-02-16T12:28:48.169394309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:28:55.848029916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:28:55.84969684Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:29:01.018996855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:29:18.173027639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
ts=2026-02-16T12:29:43.161397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:60180"
ts=2026-02-16T12:29:48.176675263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:29:50.613025039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T12:30:01.019472388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.59:11211 count=1
ts=2026-02-16T12:30:06.682174101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.54:45344"
ts=2026-02-16T12:30:13.164709323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.55:52138"
ts=2026-02-16T12:30:18.180458842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9b9f65b4 10.130.0.55:7946"
level=debug ts=2026-02-16T12:30:30.695712241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"