level=info ts=2026-01-21T21:41:06.194102761Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-01-21T21:41:06.194136931Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-01-21T21:41:06.195619602Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-01-21T21:41:06.196333061Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-01-21T21:41:06.196417046Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.66 inf=eth0
level=info ts=2026-01-21T21:41:06.196916442Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-7c6f886674-8c9s4-ad8640ab
ts=2026-01-21T21:41:06.196982922Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-01-21T21:41:06.196994558Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=debug ts=2026-01-21T21:41:06.197013747Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.66 advertisePort=7946
level=info ts=2026-01-21T21:41:06.197113241Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-01-21T21:41:06.201473571Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-01-21T21:41:06.201501804Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-01-21T21:41:06.20331429Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-01-21T21:41:06.203338072Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-01-21T21:41:06.203354543Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.243667ms
level=error ts=2026-01-21T21:41:06.203383265Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-01-21T21:41:06.203390687Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-01-21T21:41:06.206560007Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:06.206579052Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-01-21T21:41:06.206997491Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:06.207015436Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-01-21T21:41:06.207278215Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-01-21T21:41:06.207520229Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-01-21T21:41:06.207560746Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=info ts=2026-01-21T21:41:06.223580797Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-01-21T21:41:06.223552488Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-01-21T21:41:06.224217584Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=info ts=2026-01-21T21:41:06.224224948Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-01-21T21:41:06.224242591Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-01-21T21:41:06.224278529Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2026-01-21T21:41:06.224348823Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-01-21T21:41:06.224374352Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-01-21T21:41:06.224383584Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-01-21T21:41:06.224395125Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-01-21T21:41:06.224404363Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-01-21T21:41:06.224414394Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-01-21T21:41:06.224524834Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-01-21T21:41:06.224534483Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-01-21T21:41:06.224557774Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-01-21T21:41:06.224583976Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-01-21T21:41:06.224590819Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-01-21T21:41:06.22459324Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=debug ts=2026-01-21T21:41:06.224597277Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-01-21T21:41:06.224602306Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-01-21T21:41:06.224611612Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-01-21T21:41:06.224614714Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=info ts=2026-01-21T21:41:06.224617674Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-01-21T21:41:06.224625399Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-01-21T21:41:06.224630367Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-01-21T21:41:06.224634612Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-01-21T21:41:06.22463909Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2026-01-21T21:41:06.224642965Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-01-21T21:41:06.224647336Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-01-21T21:41:06.22465186Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=info ts=2026-01-21T21:41:06.224656478Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-01-21T21:41:06.224709836Z caller=loki.go:599 msg="Loki started" startup_time=89.500906ms
level=warn ts=2026-01-21T21:41:06.228040919Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-01-21T21:41:07.450603424Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:07.450627832Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-01-21T21:41:09.23095891Z caller=dns_resolver.go:225 msg="failed DNS A record lookup" err="lookup loki-query-scheduler.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-01-21T21:41:10.337482889Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:10.337517029Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-01-21T21:41:11.205578224Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:11.205604773Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-01-21T21:41:12.233242032Z caller=worker.go:235 component=querier msg="adding connection" addr=10.130.0.79:9095
ts=2026-01-21T21:41:16.041649077Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:16.041674613Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-01-21T21:41:16.205129075Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:16.20515409Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-01-21T21:41:21.20471524Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-01-21T21:41:21.204742478Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-01-21T21:41:21.573921308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-01-21T21:41:26.15191281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55800"
ts=2026-01-21T21:41:31.910417063Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-01-21T21:41:31.911796259Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.81:7946: dial tcp 10.130.0.81:7946: connect: connection refused"
ts=2026-01-21T21:41:31.912056402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-01-21T21:41:31.913514756Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
ts=2026-01-21T21:41:31.913617642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-01-21T21:41:31.913667608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45872"
ts=2026-01-21T21:41:31.916144567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.69:7946"
level=info ts=2026-01-21T21:41:31.91790643Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=25.714510926s
ts=2026-01-21T21:41:33.968318753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59936"
ts=2026-01-21T21:41:34.074637253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54314"
level=debug ts=2026-01-21T21:41:39.380795024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-01-21T21:41:52.415240431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:42:01.294694949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-01-21T21:42:06.203518968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T21:42:22.251098418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-01-21T21:42:22.416903083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:42:52.418650355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:42:56.206299974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-01-21T21:42:56.207825211Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T21:43:05.916971397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59364"
level=debug ts=2026-01-21T21:43:06.203089358Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:43:17.931982532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38054"
ts=2026-01-21T21:43:22.420390941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:43:33.485395803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-01-21T21:43:35.509531445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:58542"
ts=2026-01-21T21:43:47.93333037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57536"
ts=2026-01-21T21:43:52.421647404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:44:06.202604803Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:44:17.93499704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45046"
ts=2026-01-21T21:44:22.423344262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:44:48.279370182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-01-21T21:44:52.424575352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:45:06.203242973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:45:17.940239449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37036"
ts=2026-01-21T21:45:22.426607392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:45:35.521567754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42334"
ts=2026-01-21T21:45:47.942020534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60314"
ts=2026-01-21T21:45:52.428767757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:45:55.463310316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-01-21T21:46:06.20309941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:46:17.942954607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60688"
ts=2026-01-21T21:46:22.431090221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:46:52.43355765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:46:59.05775883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-01-21T21:47:06.203029457Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:47:22.434594131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:47:35.532297021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34868"
ts=2026-01-21T21:47:35.936192595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53206"
ts=2026-01-21T21:47:47.947332245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45628"
ts=2026-01-21T21:47:52.436556116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T21:48:03.749820818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-01-21T21:48:03.75138012Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-01-21T21:48:06.203118422Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:48:22.438580467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:48:52.440044824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:48:56.634283257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:49:05.941339717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52908"
level=debug ts=2026-01-21T21:49:06.20292619Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:49:22.441754047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T21:49:35.942918841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39224"
level=debug ts=2026-01-21T21:49:38.579756228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:49:52.443612283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T21:50:06.203504052Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:50:22.445430554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:50:28.763132716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:50:35.944642613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50320"
ts=2026-01-21T21:50:52.446305774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T21:51:05.552484355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34214"
level=debug ts=2026-01-21T21:51:06.203011508Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:51:22.448677055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T21:51:29.107247839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:51:47.961922942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33396"
ts=2026-01-21T21:51:52.451528024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:52:05.949549026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36940"
level=debug ts=2026-01-21T21:52:06.121368562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T21:52:06.123518579Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-01-21T21:52:06.203196613Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:52:22.45463151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:52:35.562588559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36744"
level=debug ts=2026-01-21T21:52:51.779562403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:52:52.456894479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T21:53:05.951886194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51718"
level=debug ts=2026-01-21T21:53:06.203022057Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:53:22.458539651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:53:32.213842519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:53:35.568146661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39660"
ts=2026-01-21T21:53:52.460379826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:54:05.571132969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51774"
ts=2026-01-21T21:54:05.956489814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40314"
level=debug ts=2026-01-21T21:54:06.203438292Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T21:54:12.03376609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:54:17.972861776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55518"
ts=2026-01-21T21:54:22.462830731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:54:35.574252708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39910"
level=debug ts=2026-01-21T21:54:48.978866101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:54:52.464047525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T21:55:05.578385087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52576"
ts=2026-01-21T21:55:05.960933937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57826"
level=debug ts=2026-01-21T21:55:06.203040426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:55:17.977938817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56130"
ts=2026-01-21T21:55:22.465281597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:55:32.484119496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T21:55:32.486011886Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T21:55:52.466374777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:56:06.20294118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T21:56:09.42559731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:56:22.468473722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:56:52.471303092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T21:57:05.591006277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54584"
level=debug ts=2026-01-21T21:57:06.202375284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T21:57:07.579611678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:57:22.473386203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:57:44.76428147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:57:47.998370788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43588"
ts=2026-01-21T21:57:52.477254133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T21:58:06.203504131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T21:58:22.479479864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T21:58:22.801554384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T21:58:35.980287456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42584"
ts=2026-01-21T21:58:52.481671467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:59:05.605291633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:48130"
ts=2026-01-21T21:59:05.992300753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43734"
level=debug ts=2026-01-21T21:59:06.212681279Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T21:59:18.065632801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T21:59:18.06823669Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T21:59:22.483366194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
ts=2026-01-21T21:59:35.993718561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33284"
ts=2026-01-21T21:59:52.485577336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:00:05.610787723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53376"
level=debug ts=2026-01-21T22:00:06.204374472Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:00:13.743618963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:00:22.488802922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:00:35.998382278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56250"
ts=2026-01-21T22:00:52.491384552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:01:06.202885866Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:01:09.978494892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:01:22.492719642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:01:36.002969497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51188"
ts=2026-01-21T22:01:52.49383304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T22:01:58.566086634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-01-21T22:02:06.203436467Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:02:22.496497614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:02:35.626080703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52406"
ts=2026-01-21T22:02:52.498573091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:02:53.023108261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:03:05.629052231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38094"
level=debug ts=2026-01-21T22:03:06.203240934Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:03:22.49983161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T22:03:31.754034624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:03:31.756232688Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:03:48.026018088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46748"
ts=2026-01-21T22:03:52.501112791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:04:06.203579672Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:04:22.503448237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:04:29.825347974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:04:52.507558882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7ce1067d 10.129.0.69:7946"
level=debug ts=2026-01-21T22:05:06.202977342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:05:14.413253381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38482"
ts=2026-01-21T22:05:14.425594487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38498"
level=debug ts=2026-01-21T22:05:20.35537432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:05:22.509886132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9ae883c3 10.129.0.69:7946"
ts=2026-01-21T22:05:33.19881044Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7ce1067d (timeout reached)"
ts=2026-01-21T22:05:36.198705816Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7ce1067d has failed, no acks received"
ts=2026-01-21T22:05:38.19960433Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7ce1067d (timeout reached)"
ts=2026-01-21T22:05:41.199306995Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7ce1067d has failed, no acks received"
ts=2026-01-21T22:05:48.044941606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39368"
ts=2026-01-21T22:05:52.51285189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:06:06.20282284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:06:22.306239132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:06:22.515206048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:06:35.651883816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44192"
ts=2026-01-21T22:06:52.516991139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:07:05.655436426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44480"
level=debug ts=2026-01-21T22:07:06.2028806Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:07:16.695970821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:07:22.519123153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:07:48.054624109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38296"
ts=2026-01-21T22:07:52.522335716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:08:05.66232564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:48990"
level=debug ts=2026-01-21T22:08:06.203145633Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:08:06.984418945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:08:06.98630529Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:08:22.524044084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:08:52.52561872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:08:56.353707486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-01-21T22:09:06.20337552Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:09:13.368684269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35884"
ts=2026-01-21T22:09:18.06020873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59616"
ts=2026-01-21T22:09:22.527118354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:09:36.235392635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:09:52.52964583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:10:06.202924493Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:10:22.531840609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9ae883c3 10.129.0.69:7946"
level=debug ts=2026-01-21T22:10:34.15843831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:10:48.066348065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47602"
ts=2026-01-21T22:10:52.53378706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:11:05.681477076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:41674"
level=debug ts=2026-01-21T22:11:06.204337065Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:11:13.378665281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40078"
ts=2026-01-21T22:11:18.068098414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35754"
ts=2026-01-21T22:11:22.535761553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:11:31.958857842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:11:48.06958477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:32940"
ts=2026-01-21T22:11:52.537354434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9ae883c3 10.129.0.69:7946"
level=debug ts=2026-01-21T22:12:06.203644438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:12:07.49486669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:12:07.496904336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:12:22.54030478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:12:52.544117874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9ae883c3 10.129.0.69:7946"
ts=2026-01-21T22:13:05.701534721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33400"
level=debug ts=2026-01-21T22:13:06.20265972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:13:07.591871859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:13:13.387532682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42578"
ts=2026-01-21T22:13:22.545457308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:13:52.5473905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:13:55.921104803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-01-21T22:14:06.20346976Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:14:18.07897825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57036"
ts=2026-01-21T22:14:18.197735768Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9ae883c3 (timeout reached)"
ts=2026-01-21T22:14:21.197688684Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9ae883c3 has failed, no acks received"
ts=2026-01-21T22:14:22.548361585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:14:24.56869285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52192"
ts=2026-01-21T22:14:24.578560371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52208"
ts=2026-01-21T22:14:35.711290322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35952"
ts=2026-01-21T22:14:41.198045545Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-9ae883c3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-01-21T22:14:43.198504922Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9ae883c3 (timeout reached)"
ts=2026-01-21T22:14:46.19904288Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9ae883c3 has failed, no acks received"
level=debug ts=2026-01-21T22:14:47.462015552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:14:52.550989976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:15:06.203355816Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:15:18.081494935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53156"
ts=2026-01-21T22:15:22.554427117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:15:33.109848441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:15:52.557976801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:16:05.720836987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36398"
level=debug ts=2026-01-21T22:16:06.202823107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:16:22.561155597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:16:23.293433522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:16:23.295246183Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:16:48.087785551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50450"
ts=2026-01-21T22:16:52.562864711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:16:59.235689346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57368"
level=debug ts=2026-01-21T22:17:02.76391168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-01-21T22:17:06.203311221Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:17:22.56392238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:17:46.197712969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:17:52.566061132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:17:59.238200989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47978"
level=debug ts=2026-01-21T22:18:06.202508709Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:18:18.094386505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52274"
ts=2026-01-21T22:18:22.569550895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:18:27.193053917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:18:35.740097155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:48938"
ts=2026-01-21T22:18:52.571486587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:19:05.743261093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40190"
level=debug ts=2026-01-21T22:19:06.202740424Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:19:17.671543237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:19:22.572432325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:19:35.746215871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51994"
ts=2026-01-21T22:19:48.102880558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54940"
ts=2026-01-21T22:19:52.574719084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:20:06.20326211Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:20:11.50709201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:20:11.508651314Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:20:18.103831631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49314"
ts=2026-01-21T22:20:22.577419194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:20:51.944553197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:20:52.57949214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:21:05.756797578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42956"
level=debug ts=2026-01-21T22:21:06.203261322Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:21:22.58135909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:21:29.253401665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44346"
level=debug ts=2026-01-21T22:21:29.60286344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:21:48.110646305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49108"
ts=2026-01-21T22:21:52.582807228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:22:05.763379774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:58926"
level=debug ts=2026-01-21T22:22:06.203128024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:22:18.11201661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33402"
ts=2026-01-21T22:22:22.585468813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:22:22.761188765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:22:52.587043067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:22:59.256770736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41682"
level=debug ts=2026-01-21T22:23:03.92678255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:23:05.768878225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46648"
level=debug ts=2026-01-21T22:23:06.203265356Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:23:22.589692922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:23:43.099248842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:23:43.10075307Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:23:52.591559226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:24:06.203373418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:24:22.593140002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:24:45.353685682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:24:52.595521918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:25:06.203339904Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:25:18.122793536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43378"
ts=2026-01-21T22:25:22.598441497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:25:37.40844017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:25:52.600107592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:26:06.203197483Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:26:22.601134281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:26:30.819635718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:26:48.127889537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60816"
ts=2026-01-21T22:26:52.603660378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:26:59.273522209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60022"
level=debug ts=2026-01-21T22:27:06.203247961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:27:08.835323882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:27:18.129190128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59726"
ts=2026-01-21T22:27:22.605627712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
ts=2026-01-21T22:27:52.606670628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-55cc5fc448-kwd8z-996e01ab 10.129.0.67:7946"
level=debug ts=2026-01-21T22:28:06.202660606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:28:09.345040189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:28:09.346701197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:28:22.608099978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:28:35.801116538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47352"
level=debug ts=2026-01-21T22:28:49.78410287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:28:52.610383581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:29:06.203250849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:29:22.612743552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:29:37.777659804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:29:52.614040781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
level=debug ts=2026-01-21T22:30:06.202811559Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
ts=2026-01-21T22:30:18.139599224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45804"
ts=2026-01-21T22:30:22.61593697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:30:25.23528688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:30:29.291305155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33426"
ts=2026-01-21T22:30:52.619029402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
level=debug ts=2026-01-21T22:31:06.202796216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-01-21T22:31:12.590356061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-01-21T22:31:18.142206074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37152"
ts=2026-01-21T22:31:22.622259108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2ff2c924 10.128.0.69:7946"
ts=2026-01-21T22:31:52.624491567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-549d79da 10.129.0.69:7946"
ts=2026-01-21T22:31:59.297254589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51812"
level=debug ts=2026-01-21T22:32:05.281776179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-01-21T22:32:05.283886659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-01-21T22:32:05.822390959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46374"
level=debug ts=2026-01-21T22:32:06.203142391Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1