level=info ts=2026-05-18T22:51:38.373233191Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-18T22:51:38.373568867Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-18T22:51:38.375378506Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-18T22:51:38.376215736Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-18T22:51:38.376295144Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.73 inf=eth0
level=info ts=2026-05-18T22:51:38.376987735Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-querier-5586f77b45-7jlt7-027313be
ts=2026-05-18T22:51:38.377075258Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-18T22:51:38.377086709Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=debug ts=2026-05-18T22:51:38.377102846Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=info ts=2026-05-18T22:51:38.377229238Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-18T22:51:38.380554695Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T22:51:38.380581182Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-18T22:51:38.382689399Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-18T22:51:38.382710886Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-18T22:51:38.382739104Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.5107ms
level=error ts=2026-05-18T22:51:38.382794676Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-18T22:51:38.382813535Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-18T22:51:38.385166554Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:38.385188696Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-18T22:51:38.385508159Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:38.385523765Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-18T22:51:38.385777248Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=debug ts=2026-05-18T22:51:38.385977938Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=true queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=false
level=info ts=2026-05-18T22:51:38.38600116Z caller=worker.go:138 component=querier msg="Starting querier worker connected to query-scheduler" scheduler=loki-query-scheduler.product-kubearchive-logging.svc.cluster.local:9095
level=debug ts=2026-05-18T22:51:38.399961141Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2026-05-18T22:51:38.399959692Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2026-05-18T22:51:38.399981643Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-05-18T22:51:38.399982428Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2026-05-18T22:51:38.399997932Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-18T22:51:38.400005658Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-18T22:51:38.400005486Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ingester-querier
level=info ts=2026-05-18T22:51:38.400016026Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-18T22:51:38.400021364Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-18T22:51:38.400042627Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-18T22:51:38.40005263Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-18T22:51:38.400057843Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2026-05-18T22:51:38.40029177Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-18T22:51:38.400325571Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-18T22:51:38.400372094Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-18T22:51:38.400420098Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=runtime-config
level=debug ts=2026-05-18T22:51:38.400427959Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2026-05-18T22:51:38.400433669Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2026-05-18T22:51:38.400442365Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-18T22:51:38.400445564Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=ring
level=debug ts=2026-05-18T22:51:38.400448961Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-18T22:51:38.400451766Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-18T22:51:38.400454386Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-18T22:51:38.400464761Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=analytics
level=debug ts=2026-05-18T22:51:38.40046848Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=debug ts=2026-05-18T22:51:38.400471637Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2026-05-18T22:51:38.400474548Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2026-05-18T22:51:38.400477137Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2026-05-18T22:51:38.400479742Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=runtime-config
level=debug ts=2026-05-18T22:51:38.400482246Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=info ts=2026-05-18T22:51:38.400484594Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2026-05-18T22:51:38.400562654Z caller=loki.go:615 msg="Loki started" startup_time=90.816702ms
level=info ts=2026-05-18T22:51:38.403928343Z caller=worker.go:235 component=querier msg="adding connection" addr=172.30.3.118:9095
level=warn ts=2026-05-18T22:51:38.404937376Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
level=warn ts=2026-05-18T22:51:38.404944204Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
level=warn ts=2026-05-18T22:51:39.165897879Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
level=warn ts=2026-05-18T22:51:39.382934674Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
ts=2026-05-18T22:51:39.608118741Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:39.608150408Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-18T22:51:40.323804328Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
level=warn ts=2026-05-18T22:51:40.525470666Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
ts=2026-05-18T22:51:42.726838372Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:42.726863316Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-18T22:51:42.887441674Z caller=scheduler_processor.go:103 component=querier msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 172.30.3.118:9095: connect: connection refused\"" addr=172.30.3.118:9095
level=error ts=2026-05-18T22:51:43.384000756Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg="failed to lookup IP addresses" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local err="lookup loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:43.384027133Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg="IP address lookup yielded no results. No host found or no addresses found" host=loki-index-gateway-headless.product-kubearchive-logging.svc.cluster.local
ts=2026-05-18T22:51:47.709991974Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T22:51:47.710017217Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-18T22:52:00.407147903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:52:02.994844348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.84:7946"
ts=2026-05-18T22:52:02.998319128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
ts=2026-05-18T22:52:02.999776957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-05-18T22:52:02.999839773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56676"
ts=2026-05-18T22:52:03.000801826Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.74:7946: dial tcp 10.130.0.74:7946: connect: connection refused"
ts=2026-05-18T22:52:03.001627042Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused"
ts=2026-05-18T22:52:03.002282841Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.75:7946: dial tcp 10.130.0.75:7946: connect: connection refused"
level=info ts=2026-05-18T22:52:03.002292465Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.619471393s
level=debug ts=2026-05-18T22:52:19.430547626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:52:23.315574031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T22:52:34.905971583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-18T22:52:38.382402787Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:52:53.318899879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T22:52:55.266895903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40322"
level=debug ts=2026-05-18T22:53:00.4945951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:53:19.251284026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50504"
ts=2026-05-18T22:53:23.321419413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T22:53:25.269237711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38084"
level=debug ts=2026-05-18T22:53:28.929656788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-18T22:53:28.931182387Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T22:53:38.381794773Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:53:53.324094906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T22:54:05.811945436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:54:23.326709626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T22:54:25.273419386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51648"
ts=2026-05-18T22:54:31.885658648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45140"
level=debug ts=2026-05-18T22:54:38.381660096Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:54:53.329605393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T22:55:06.388448495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:55:23.332904567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T22:55:25.278809058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46510"
level=debug ts=2026-05-18T22:55:38.381717265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:55:49.259250152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56896"
ts=2026-05-18T22:55:53.335768211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T22:56:00.754419983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:56:19.262055265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54604"
ts=2026-05-18T22:56:23.338239268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T22:56:31.892339913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:36734"
level=debug ts=2026-05-18T22:56:38.382391944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:56:53.340976683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T22:56:55.285280897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58626"
level=debug ts=2026-05-18T22:57:07.674393568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T22:57:23.343840213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T22:57:38.381977899Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:57:53.346391607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T22:57:55.934937122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-18T22:57:55.936386179Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T22:58:01.898218754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50570"
ts=2026-05-18T22:58:19.268695572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51496"
ts=2026-05-18T22:58:23.3498538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T22:58:31.901521226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49752"
level=debug ts=2026-05-18T22:58:38.087411236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T22:58:38.382407088Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:58:49.271731168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34900"
ts=2026-05-18T22:58:53.35263654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T22:59:01.904698115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37226"
ts=2026-05-18T22:59:23.355912867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T22:59:27.086826708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T22:59:31.907697205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45424"
level=debug ts=2026-05-18T22:59:38.381916195Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T22:59:49.276211119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60222"
ts=2026-05-18T22:59:53.358567091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:00:01.910299491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45040"
level=debug ts=2026-05-18T23:00:11.998983759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:00:23.36161934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:00:31.912846891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45098"
level=debug ts=2026-05-18T23:00:38.381785579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:00:49.279983708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41660"
ts=2026-05-18T23:00:53.364798691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:01:10.683700536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:01:19.283031574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37034"
ts=2026-05-18T23:01:23.36767587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:01:38.382382336Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:01:49.28543032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50618"
level=debug ts=2026-05-18T23:01:50.391760521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:01:50.393502609Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:01:53.370878894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:01:55.300551814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34732"
ts=2026-05-18T23:02:01.918982921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53196"
ts=2026-05-18T23:02:23.374404348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:02:31.92200982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51630"
level=debug ts=2026-05-18T23:02:38.381656181Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:02:39.370967709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:02:53.377477374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:03:01.92463231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42202"
ts=2026-05-18T23:03:23.380167075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:03:25.30528215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43566"
ts=2026-05-18T23:03:31.927537735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34880"
level=debug ts=2026-05-18T23:03:38.167188394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:03:38.381881061Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:03:49.292855548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:46724"
ts=2026-05-18T23:03:53.382685437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:04:20.442836094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:04:23.385123142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:04:25.30893452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41258"
level=debug ts=2026-05-18T23:04:38.381836128Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:04:53.387401428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:04:55.311887181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58922"
level=debug ts=2026-05-18T23:04:57.974654258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:05:23.390526614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:05:25.315288289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51008"
level=debug ts=2026-05-18T23:05:37.445881171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:05:37.447754986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:05:38.38161435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:05:49.299383325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44724"
ts=2026-05-18T23:05:53.393881017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:05:55.3182289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46336"
ts=2026-05-18T23:06:23.396531422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:06:25.32149424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40632"
level=debug ts=2026-05-18T23:06:30.662000303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:06:31.936035708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40046"
level=debug ts=2026-05-18T23:06:38.381970103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:06:53.399584276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:07:01.939173557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38704"
ts=2026-05-18T23:07:19.305133518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37758"
level=debug ts=2026-05-18T23:07:19.808790526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:07:23.402390192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:07:38.382516779Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:07:53.405264285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:08:15.678774221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:08:19.309227558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56022"
ts=2026-05-18T23:08:23.407647223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:08:38.382463397Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:08:53.410662361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:08:59.438260682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:09:23.413522094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:09:38.382144327Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:09:50.82113724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:09:50.822799285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:09:53.416739362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:09:55.333481061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42092"
ts=2026-05-18T23:10:23.419678929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:10:30.767745736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:10:38.381662204Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:10:53.422913804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:11:08.643478018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:11:23.425784287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:11:25.340409763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36306"
level=debug ts=2026-05-18T23:11:38.382589876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:11:53.428257068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:11:55.105843453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:12:23.431481897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:12:31.954004258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38990"
level=debug ts=2026-05-18T23:12:38.381802552Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:12:49.322729946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:48244"
level=debug ts=2026-05-18T23:12:50.711448367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:12:53.434057298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:13:23.437158831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:13:26.816462542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:13:26.818085532Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:13:38.38182894Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:13:53.439796881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:14:15.531356043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:14:23.442587694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:14:38.381508268Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:14:49.330501757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45654"
level=debug ts=2026-05-18T23:14:50.76029978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:14:53.445903703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:15:23.448947298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:15:31.963168727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:33724"
level=debug ts=2026-05-18T23:15:38.382017989Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:15:50.225155981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:15:53.451930786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:15:55.353868784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48128"
ts=2026-05-18T23:16:01.966427688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39740"
ts=2026-05-18T23:16:23.454617672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:16:35.404198483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:16:38.382581314Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:16:49.337539039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36710"
ts=2026-05-18T23:16:53.457516279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:16:55.358169354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40966"
level=debug ts=2026-05-18T23:17:16.195443221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:17:16.197111005Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:17:23.460204966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:17:38.381555285Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:17:50.40353617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:17:53.462825327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:17:55.362887472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47348"
ts=2026-05-18T23:18:01.972202506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59546"
ts=2026-05-18T23:18:23.465690382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:18:26.941364694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:18:38.382933307Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:18:49.345724957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:35952"
ts=2026-05-18T23:18:53.469256424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:19:17.6100219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:19:23.472789616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:19:38.38238717Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:19:49.349736333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38530"
ts=2026-05-18T23:19:53.475536509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:19:55.371031418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:44782"
level=debug ts=2026-05-18T23:19:58.156680094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:20:23.478801595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:20:38.382166024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:20:53.48146807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:20:56.733622259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:20:56.735113876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:21:19.354163156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59518"
ts=2026-05-18T23:21:23.48429288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:21:38.381842246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:21:49.357016838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59630"
level=debug ts=2026-05-18T23:21:52.1526188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:21:53.487760117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:22:23.490375868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:22:38.382162994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:22:44.398510864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:22:49.360566419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51736"
ts=2026-05-18T23:22:53.493469188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:23:23.495871273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:23:38.381583519Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:23:43.30558981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:23:49.36496727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37666"
ts=2026-05-18T23:23:53.498705376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:24:01.991127701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52736"
ts=2026-05-18T23:24:19.368047247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:44388"
level=debug ts=2026-05-18T23:24:21.996163797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:24:23.501650503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:24:31.993991821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60594"
level=debug ts=2026-05-18T23:24:38.382083359Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:24:53.504855733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:24:55.386108864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52984"
level=debug ts=2026-05-18T23:25:06.02817483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:25:06.02963136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:25:23.507319351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:25:25.389164856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58364"
level=debug ts=2026-05-18T23:25:38.38164257Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:25:50.68636744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:25:53.509664667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:25:55.391759043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48482"
ts=2026-05-18T23:26:23.512879632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:26:25.39511995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:60202"
level=debug ts=2026-05-18T23:26:27.468873886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:26:38.382255557Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:26:49.376819228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:36688"
ts=2026-05-18T23:26:53.515713816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:27:23.518938822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:27:24.165863064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:27:32.002933929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48218"
level=debug ts=2026-05-18T23:27:38.381631724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:27:53.521468265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:27:55.400223914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58254"
level=debug ts=2026-05-18T23:28:01.405348663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:28:19.382091021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:33470"
ts=2026-05-18T23:28:23.524184985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:28:25.40332897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42482"
level=debug ts=2026-05-18T23:28:38.382429421Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:28:48.509824075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:28:48.51167175Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:28:49.385058844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:58546"
ts=2026-05-18T23:28:53.526549877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:29:02.008130499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44870"
ts=2026-05-18T23:29:23.529080465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:29:38.382544019Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:29:46.758271119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:29:53.531828906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:30:23.53488784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:30:27.020342496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:30:32.013553259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45266"
level=debug ts=2026-05-18T23:30:38.382356221Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:30:53.537854083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:31:16.459916646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:31:23.540938043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:31:38.382333762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:31:53.543946921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:31:55.413742187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38268"
level=debug ts=2026-05-18T23:31:58.291445174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:32:19.395152303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59320"
ts=2026-05-18T23:32:23.546409622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:32:25.416908999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45600"
level=debug ts=2026-05-18T23:32:38.381654192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:32:38.731361163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:32:38.733045124Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:32:53.549334419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:33:02.022394892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49670"
level=debug ts=2026-05-18T23:33:18.965454038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:33:23.55193684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:33:25.421251037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48954"
ts=2026-05-18T23:33:32.024969765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39880"
level=debug ts=2026-05-18T23:33:38.382266166Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:33:53.554902561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:33:58.160755347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:34:23.557224512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:34:38.381948734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:34:41.481941027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:34:53.560486697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:34:55.42687255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41790"
ts=2026-05-18T23:35:19.404597314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39316"
ts=2026-05-18T23:35:23.563248133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:35:30.817704117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:35:32.032242114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50782"
level=debug ts=2026-05-18T23:35:38.382276794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:35:53.566773922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:35:55.431221596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34896"
ts=2026-05-18T23:36:02.035363319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44072"
level=debug ts=2026-05-18T23:36:18.770544901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:36:18.772147839Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:36:23.569295556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:36:38.381902929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:36:53.572198347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:36:59.744297955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:37:02.038791272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44448"
ts=2026-05-18T23:37:19.412109792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38896"
ts=2026-05-18T23:37:23.575608753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:37:25.437151413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36224"
ts=2026-05-18T23:37:32.041913517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39810"
level=debug ts=2026-05-18T23:37:38.381641129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:37:49.414720824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38278"
ts=2026-05-18T23:37:53.578830557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:37:53.763113081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:38:02.045156695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34064"
ts=2026-05-18T23:38:23.582025884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:38:25.441496547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55288"
ts=2026-05-18T23:38:32.047482639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44614"
level=debug ts=2026-05-18T23:38:38.382325952Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:38:49.418591734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51504"
level=debug ts=2026-05-18T23:38:52.272638338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:38:53.58510084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:39:19.421962336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51264"
ts=2026-05-18T23:39:23.587505704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:39:38.381975238Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:39:40.123675552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:39:53.590362949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:39:55.446828215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48148"
ts=2026-05-18T23:40:19.42581985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41894"
ts=2026-05-18T23:40:23.593729031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:40:25.449800575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38222"
ts=2026-05-18T23:40:32.054570687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59720"
level=debug ts=2026-05-18T23:40:34.357393755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:40:34.359074318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:40:38.381781252Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:40:49.428990839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40352"
ts=2026-05-18T23:40:53.596911877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:40:55.453143591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47856"
level=debug ts=2026-05-18T23:41:16.04341508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:41:23.599839543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:41:25.455531536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58374"
ts=2026-05-18T23:41:32.058982684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35036"
level=debug ts=2026-05-18T23:41:38.381807779Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:41:53.602895943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:41:57.095206193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:42:23.605328661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:42:32.064021507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53328"
level=debug ts=2026-05-18T23:42:38.382115289Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:42:53.608240275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:42:57.751691539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:43:23.610688482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:43:36.608877541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:43:38.381743674Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:43:49.438901221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:47576"
ts=2026-05-18T23:43:53.613575972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:43:55.463178594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49872"
ts=2026-05-18T23:44:02.069373125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48166"
ts=2026-05-18T23:44:23.615962431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:44:36.356778376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:44:36.358381373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:44:38.382073591Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:44:49.443047796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:60278"
ts=2026-05-18T23:44:53.618468868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:45:02.072878861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49356"
level=debug ts=2026-05-18T23:45:11.491058527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:45:23.62153505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:45:38.381543365Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:45:49.446667471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:55036"
ts=2026-05-18T23:45:53.623977871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:45:55.928367637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:46:02.077431284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45114"
ts=2026-05-18T23:46:19.449039949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59336"
ts=2026-05-18T23:46:23.627236661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-18T23:46:25.472472835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47824"
level=debug ts=2026-05-18T23:46:38.382199795Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:46:48.759599898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:46:53.630069021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:46:55.475437819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58418"
ts=2026-05-18T23:47:02.082478825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59920"
ts=2026-05-18T23:47:23.633418008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:47:32.734143739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:47:38.382423765Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:47:53.635993929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:48:23.639410353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:48:30.274933693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:48:30.276511404Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T23:48:32.088278992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48220"
level=debug ts=2026-05-18T23:48:38.381865799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:48:49.45815753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41980"
ts=2026-05-18T23:48:53.642585609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:49:23.645940029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:49:27.99926723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:49:32.093391307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55784"
level=debug ts=2026-05-18T23:49:38.38198976Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:49:53.649152128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:50:13.058213119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:50:19.464584587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38238"
ts=2026-05-18T23:50:23.652037857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:50:25.48529141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:44656"
ts=2026-05-18T23:50:32.0979551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41040"
level=debug ts=2026-05-18T23:50:38.381920622Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:50:49.467541945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:43844"
ts=2026-05-18T23:50:53.655409163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:51:05.001528541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:51:19.470723919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41556"
ts=2026-05-18T23:51:23.658070718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:51:38.3820839Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:51:46.083745225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:51:53.660597898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:52:23.663914364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-18T23:52:30.156828697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:52:30.158398553Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:52:38.381630047Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:52:53.667069805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:53:23.669957765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:53:25.993182087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:53:38.381691791Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:53:53.672999283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:54:02.108745152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45396"
level=debug ts=2026-05-18T23:54:09.936055398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:54:23.676165595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:54:25.497900869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45310"
level=debug ts=2026-05-18T23:54:38.381524579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:54:48.076617156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:54:53.67924127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:55:23.681861574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:55:25.502352736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42956"
level=debug ts=2026-05-18T23:55:38.381692282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-18T23:55:45.092940999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:55:53.684286238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:56:19.48503625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39474"
ts=2026-05-18T23:56:23.687624932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:56:32.165711773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T23:56:32.167368907Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T23:56:38.381905233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:56:49.487768541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:38932"
ts=2026-05-18T23:56:53.690438753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:56:55.507911919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39014"
ts=2026-05-18T23:57:02.118371446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45912"
ts=2026-05-18T23:57:23.693690769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-18T23:57:28.165463434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:57:38.381550018Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:57:53.696425924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-18T23:57:55.512414903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38052"
level=debug ts=2026-05-18T23:58:15.446550599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:58:23.69949428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:58:25.515659389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46170"
level=debug ts=2026-05-18T23:58:38.381857066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:58:53.702466663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:58:57.264226446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T23:59:23.705237267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-18T23:59:33.253290672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-18T23:59:38.382371569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-18T23:59:49.497479674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39696"
ts=2026-05-18T23:59:53.708460685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-18T23:59:55.520680863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54098"
ts=2026-05-19T00:00:02.127214561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47448"
ts=2026-05-19T00:00:23.711371605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:00:32.129978445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51452"
level=debug ts=2026-05-19T00:00:34.337867335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:00:34.340093999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T00:00:38.382230882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:00:53.714654023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:00:55.5259362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58410"
ts=2026-05-19T00:01:02.133080201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44594"
level=debug ts=2026-05-19T00:01:17.006924073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:01:23.717238002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:01:32.13606236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60842"
level=debug ts=2026-05-19T00:01:38.381688979Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:01:53.719931692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:01:55.530741151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34676"
level=debug ts=2026-05-19T00:02:10.92854053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:02:23.722701629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:02:25.534023397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35534"
level=debug ts=2026-05-19T00:02:38.382543972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:02:47.953852618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:02:49.50938955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51274"
ts=2026-05-19T00:02:53.725158462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:03:02.141859436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52956"
ts=2026-05-19T00:03:19.512632405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:37326"
ts=2026-05-19T00:03:23.72813339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:03:32.144616392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45366"
level=debug ts=2026-05-19T00:03:38.381959454Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:03:44.930176347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:03:53.731213771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:04:23.04390639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:04:23.045505026Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:04:23.733664131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:04:25.540892688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48358"
level=debug ts=2026-05-19T00:04:38.382249652Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:04:53.735952034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:05:07.372163383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:05:23.739179141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:05:32.151336649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58880"
level=debug ts=2026-05-19T00:05:38.381668911Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:05:44.177530924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:05:49.521262053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:39710"
ts=2026-05-19T00:05:53.741589484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:06:02.154502829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43894"
ts=2026-05-19T00:06:19.530573243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:40186"
ts=2026-05-19T00:06:23.744300473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:06:24.618926467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:06:38.3823884Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:06:53.747594704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:07:05.760509145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:07:23.750076113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:07:32.158791385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38018"
level=debug ts=2026-05-19T00:07:38.382351725Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:07:53.752884351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:08:05.468326268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:08:05.46998146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:08:23.755397626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:08:32.163120645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58454"
level=debug ts=2026-05-19T00:08:38.381659264Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:08:53.757836506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:08:55.555968188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47552"
level=debug ts=2026-05-19T00:09:03.472176785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:09:19.542144946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59974"
ts=2026-05-19T00:09:23.760616699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:09:25.559068133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39040"
level=debug ts=2026-05-19T00:09:38.381873491Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:09:49.544872211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:45750"
ts=2026-05-19T00:09:53.763711535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:09:55.561436807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38022"
level=debug ts=2026-05-19T00:10:00.107283448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:10:23.766853501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:10:32.169635887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43858"
level=debug ts=2026-05-19T00:10:34.44275558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:10:38.381698142Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:10:49.548442094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:52670"
ts=2026-05-19T00:10:53.769767784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:11:12.227369496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:11:19.551445583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:41310"
ts=2026-05-19T00:11:23.773159596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:11:38.382593564Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:11:49.554446916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:56908"
ts=2026-05-19T00:11:53.776105235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:11:55.568099587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42802"
level=debug ts=2026-05-19T00:12:12.018727872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:12:12.02039938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:12:23.778702865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:12:25.571197758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35510"
ts=2026-05-19T00:12:32.177492283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42346"
level=debug ts=2026-05-19T00:12:38.38205022Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:12:49.514104862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:12:53.781423414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:13:02.180597535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40078"
ts=2026-05-19T00:13:23.784506114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:13:24.172100602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:13:38.381893595Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:13:53.787598639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:14:13.32751155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:14:23.79012333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:14:38.381590036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:14:53.792663339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:15:06.560403047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:15:23.794963378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:15:38.381556146Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:15:49.567389919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:57222"
ts=2026-05-19T00:15:53.797262927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:15:55.5827281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39694"
level=debug ts=2026-05-19T00:16:00.789859945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:16:00.791353939Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:16:23.799925839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:16:38.382499767Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:16:44.164794986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:16:49.572526146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54292"
ts=2026-05-19T00:16:53.803000406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:17:02.192901807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38176"
ts=2026-05-19T00:17:23.80603536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:17:32.195709945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40644"
level=debug ts=2026-05-19T00:17:37.287023556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:17:38.382381469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:17:49.576544557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59212"
ts=2026-05-19T00:17:53.809252478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:18:23.811881148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:18:25.591381174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59766"
level=debug ts=2026-05-19T00:18:31.18419546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:18:32.200478236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51234"
level=debug ts=2026-05-19T00:18:38.382037117Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:18:53.814552957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:18:55.594506277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:32964"
ts=2026-05-19T00:19:19.582355921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:42868"
ts=2026-05-19T00:19:23.817090043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:19:32.204621275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49680"
level=debug ts=2026-05-19T00:19:32.354827849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:19:38.382465363Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:19:53.820266403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:20:07.064157686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:20:07.066121267Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:20:23.823162678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:20:32.20984464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56374"
level=debug ts=2026-05-19T00:20:38.381808968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:20:53.825933896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:20:55.602058059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33306"
level=debug ts=2026-05-19T00:21:03.576190574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:21:19.588921218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50610"
ts=2026-05-19T00:21:23.82875302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:21:25.604836512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43772"
level=debug ts=2026-05-19T00:21:38.382036972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:21:53.83119419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:21:57.083682107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:22:19.592752718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51468"
ts=2026-05-19T00:22:23.834483039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:22:25.609447754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42712"
level=debug ts=2026-05-19T00:22:38.382497228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-19T00:22:44.234590819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:22:53.836985958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
ts=2026-05-19T00:22:55.61265095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37108"
ts=2026-05-19T00:23:23.839395683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:23:32.441426154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-19T00:23:38.382069626Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:23:53.841940011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:24:23.845221081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:24:33.375156469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:24:33.376847322Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T00:24:38.382776087Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:24:53.848139634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:25:02.222584984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37470"
level=debug ts=2026-05-19T00:25:09.897332453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:25:23.850791564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:25:25.621017203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48346"
ts=2026-05-19T00:25:32.225824443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43478"
level=debug ts=2026-05-19T00:25:38.382074086Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:25:53.8533671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:26:06.027382428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:26:19.605496793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:51240"
ts=2026-05-19T00:26:23.856402925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:26:38.381711151Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:26:49.608234776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54474"
ts=2026-05-19T00:26:53.859622505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:26:55.591698438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:27:19.611260888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:59462"
ts=2026-05-19T00:27:23.862682125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
level=debug ts=2026-05-19T00:27:31.987949289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:27:32.233914289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42634"
level=debug ts=2026-05-19T00:27:38.382466033Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:27:49.61442388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:49960"
ts=2026-05-19T00:27:53.865297624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"
ts=2026-05-19T00:27:55.629520226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52754"
level=debug ts=2026-05-19T00:28:14.784777986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T00:28:14.786845633Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T00:28:19.617299994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:50720"
ts=2026-05-19T00:28:23.869560518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:28:38.381591955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:28:53.872880993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
level=debug ts=2026-05-19T00:29:15.537674497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:29:19.620750392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:54524"
ts=2026-05-19T00:29:23.875460622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ece906d 10.128.0.83:7946"
ts=2026-05-19T00:29:25.634603474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41166"
ts=2026-05-19T00:29:32.241234008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44106"
level=debug ts=2026-05-19T00:29:38.382653414Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
ts=2026-05-19T00:29:49.623058702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.83:34488"
ts=2026-05-19T00:29:53.878438174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-cae982db 10.128.0.84:7946"
level=debug ts=2026-05-19T00:29:55.381090247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T00:30:23.881066806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cl24p-5e90b45a 10.128.0.82:7946"