level=info ts=2026-05-13T20:52:03.255925606Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T20:52:03.255962412Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-13T20:52:03.257599516Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T20:52:03.257907462Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:52:03.257971041Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0
level=debug ts=2026-05-13T20:52:03.25901151Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:52:03.259111778Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.60 inf=eth0
level=info ts=2026-05-13T20:52:03.259165814Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-3cb51d11
ts=2026-05-13T20:52:03.259252015Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T20:52:03.259263793Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946
level=debug ts=2026-05-13T20:52:03.259279838Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.60 advertisePort=7946
level=info ts=2026-05-13T20:52:03.259404186Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-13T20:52:03.26633924Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T20:52:03.266354389Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:52:03.266363228Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.961698ms
level=error ts=2026-05-13T20:52:03.266371224Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T20:52:03.266382196Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-13T20:52:03.269162679Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:52:03.26918113Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:52:03.275273412Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-13T20:52:03.275275066Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T20:52:03.275297768Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-13T20:52:03.275304124Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T20:52:03.275315848Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T20:52:03.275345835Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T20:52:03.275556276Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-13T20:52:03.275590411Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-13T20:52:03.275597357Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-13T20:52:03.27560262Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-13T20:52:03.275676481Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-13T20:52:03.275667258Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-13T20:52:03.27568972Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T20:52:03.275852694Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-13T20:52:03.275867636Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-13T20:52:03.275886775Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-13T20:52:04.276515113Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-13T20:52:04.466112244Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-13T20:52:04.466166474Z caller=loki.go:615 msg="Loki started" startup_time=1.271409548s
ts=2026-05-13T20:52:05.271044791Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:52:05.271071635Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:52:08.157574817Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:52:08.157598428Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:52:09.467119421Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-13T20:52:09.467182303Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-13T20:52:15.741482361Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:52:15.741510739Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T20:52:20.779844284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:52:31.692305352Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.53:7946: dial tcp 10.129.0.53:7946: connect: connection refused"
ts=2026-05-13T20:52:31.692433063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.60:7946"
ts=2026-05-13T20:52:31.692554089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:36868"
ts=2026-05-13T20:52:31.693373895Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.58:7946: dial tcp 10.130.0.58:7946: connect: connection refused"
ts=2026-05-13T20:52:31.694611986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.61:7946"
ts=2026-05-13T20:52:31.69675067Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.64:7946: dial tcp 10.128.0.64:7946: connect: connection refused"
ts=2026-05-13T20:52:31.697625509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.62:7946"
level=info ts=2026-05-13T20:52:31.6988281Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=28.43244154s
ts=2026-05-13T20:52:36.266493624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:56234"
level=debug ts=2026-05-13T20:52:38.607401808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:52:46.156177087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36122"
ts=2026-05-13T20:52:46.747111991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:51450"
ts=2026-05-13T20:52:54.070410524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T20:52:54.848725921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:46170"
level=debug ts=2026-05-13T20:52:55.999747285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:53:06.268167125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:58526"
ts=2026-05-13T20:53:11.618768815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55800"
level=debug ts=2026-05-13T20:53:18.093830977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:53:24.072545565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T20:53:37.311318319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59270"
ts=2026-05-13T20:53:41.620698255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45342"
level=debug ts=2026-05-13T20:53:51.275430548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:53:51.277633098Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T20:53:54.074835828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T20:54:24.077078936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T20:54:33.49342278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:54:36.273200956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:34122"
ts=2026-05-13T20:54:37.315403407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40154"
ts=2026-05-13T20:54:54.079588283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T20:55:06.274961002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:44828"
ts=2026-05-13T20:55:07.317956656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:36462"
ts=2026-05-13T20:55:11.625237867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:51112"
ts=2026-05-13T20:55:24.082839399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T20:55:35.879642825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:55:54.085458088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T20:56:24.087652933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T20:56:36.282830604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:53166"
ts=2026-05-13T20:56:37.321513632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34514"
level=debug ts=2026-05-13T20:56:41.423160326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:56:54.089525204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T20:57:24.091308444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T20:57:37.324251699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57200"
ts=2026-05-13T20:57:54.093385117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T20:57:54.373837815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:58:24.095305372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T20:58:37.327284154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49414"
level=debug ts=2026-05-13T20:58:52.097743271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:58:52.09917539Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T20:58:54.097878781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T20:59:07.32907503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48596"
ts=2026-05-13T20:59:24.100193793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T20:59:47.446180873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T20:59:54.101920392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:00:06.294897183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55594"
ts=2026-05-13T21:00:24.104463143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:00:42.113443574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:00:54.10630639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:01:11.642326834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:34042"
level=debug ts=2026-05-13T21:01:16.676237394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:01:24.108124197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:01:37.336701192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:51582"
ts=2026-05-13T21:01:41.644480916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:51754"
ts=2026-05-13T21:01:54.110268253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:02:06.301206408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:48410"
level=info ts=2026-05-13T21:02:09.467361386Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-13T21:02:09.473030839Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:02:09.474840877Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:02:09.476411628Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:02:09.476436694Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:02:09.476447774Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-13T21:02:11.646347146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52526"
level=debug ts=2026-05-13T21:02:12.088155999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:02:24.112767332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:02:54.115281561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:03:04.134874244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:03:04.136752316Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:03:24.116957825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:03:54.119351753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:04:03.893895645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:04:06.308683359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:45920"
ts=2026-05-13T21:04:24.122104599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:04:39.796404451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:04:41.654176559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:56862"
ts=2026-05-13T21:04:54.124558305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:05:11.656055329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49758"
ts=2026-05-13T21:05:24.126754735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:05:29.344013137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:05:41.657834532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:41312"
ts=2026-05-13T21:05:54.128655675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:06:06.314483695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55298"
ts=2026-05-13T21:06:11.659738445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:60900"
ts=2026-05-13T21:06:24.130841003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:06:27.959442127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:06:36.316812979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:40558"
ts=2026-05-13T21:06:37.350978425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34876"
ts=2026-05-13T21:06:41.661505583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:33560"
ts=2026-05-13T21:06:54.133138882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:07:08.534456442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:07:08.536475992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:07:11.663333763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40926"
ts=2026-05-13T21:07:24.134754569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:07:37.355135442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34304"
ts=2026-05-13T21:07:54.137047711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:08:05.486170427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:08:06.321751999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55816"
ts=2026-05-13T21:08:07.357151963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48834"
ts=2026-05-13T21:08:11.667711286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:37640"
ts=2026-05-13T21:08:24.138685211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:08:37.359479211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35156"
ts=2026-05-13T21:08:41.669750612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43834"
level=debug ts=2026-05-13T21:08:44.61904136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:08:54.140841769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:09:24.143109341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:09:36.328942851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:54328"
ts=2026-05-13T21:09:37.363503733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33548"
level=debug ts=2026-05-13T21:09:40.130480031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:09:54.145703906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:10:24.147844974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:10:36.333826555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:35704"
ts=2026-05-13T21:10:37.3728227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45092"
level=debug ts=2026-05-13T21:10:40.576770801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:10:54.150308421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:11:11.682140378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:48074"
level=debug ts=2026-05-13T21:11:23.26118504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:11:23.263874155Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:11:24.152483559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:11:36.337450396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:46438"
ts=2026-05-13T21:11:54.154901236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:12:06.339476782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:34872"
level=error ts=2026-05-13T21:12:09.482520767Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:12:09.485124657Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:12:09.48735324Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:12:09.4873836Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:12:11.685131791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49698"
level=debug ts=2026-05-13T21:12:16.124711945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:12:24.156834597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:12:41.686797886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:35594"
level=debug ts=2026-05-13T21:12:50.485568974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:12:54.159079022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:13:07.381594183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47016"
ts=2026-05-13T21:13:11.688821362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45358"
ts=2026-05-13T21:13:24.161006073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:13:36.013123748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:13:36.346217008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:50732"
ts=2026-05-13T21:13:54.163479202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:14:22.548558916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:14:24.165194489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:14:54.167149658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:15:07.388247241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:36230"
level=debug ts=2026-05-13T21:15:11.042967159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:15:11.044711034Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:15:11.694861531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43266"
ts=2026-05-13T21:15:24.1690303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:15:36.351794064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:48378"
level=debug ts=2026-05-13T21:15:49.423283373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:15:54.17139987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:16:24.17340632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:16:49.396288609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:16:54.175171998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:17:06.358282536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:37704"
ts=2026-05-13T21:17:24.178233963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:17:26.833363824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:17:54.18021314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:18:06.362944046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:58418"
ts=2026-05-13T21:18:11.704649803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43654"
level=debug ts=2026-05-13T21:18:14.999484206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:18:24.18227131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:18:37.400007073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33270"
ts=2026-05-13T21:18:54.184238492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:19:08.681924941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:19:08.6835685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:19:24.186285425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:19:41.708429549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:58932"
ts=2026-05-13T21:19:54.188150398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:19:55.862482559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:20:24.190599749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:20:39.34112144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:20:54.193021124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:21:11.713872205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:60114"
ts=2026-05-13T21:21:24.195581306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:21:24.611987872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:21:54.197779126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:22:08.279667102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:22:09.479337352Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:22:09.481118083Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:22:09.482797756Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:22:09.48282719Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:22:11.717860465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55062"
ts=2026-05-13T21:22:24.19988342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:22:36.378669064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:57478"
level=debug ts=2026-05-13T21:22:52.447478793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:22:52.449208737Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:22:54.202232586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:23:24.204345681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:23:41.72241926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59702"
level=debug ts=2026-05-13T21:23:54.049592281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:23:54.206938185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:24:24.209150617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:24:36.385112405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:45188"
ts=2026-05-13T21:24:37.417084848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58478"
level=debug ts=2026-05-13T21:24:42.099745713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:24:54.211308615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:25:24.212961302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:25:27.058271813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:25:36.389520403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:39556"
ts=2026-05-13T21:25:54.214576776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:26:06.392179815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:46504"
ts=2026-05-13T21:26:07.42198915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:36756"
ts=2026-05-13T21:26:11.730126697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59808"
ts=2026-05-13T21:26:24.21657879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:26:28.662714223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:26:36.394521097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:43514"
ts=2026-05-13T21:26:37.424355976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34468"
ts=2026-05-13T21:26:41.731995529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:47890"
ts=2026-05-13T21:26:54.218912939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:27:13.458610303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:27:13.460854111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:27:24.221310724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:27:54.223639923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:27:57.289527708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:28:07.428880034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45028"
ts=2026-05-13T21:28:24.225691948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:28:46.332885662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:28:54.227509918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:29:24.230067927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:29:25.672949365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:29:54.232111853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:30:09.465970035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:30:24.234465954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:30:50.050349988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:30:50.052361728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:30:54.236626456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:31:07.438638691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:37238"
ts=2026-05-13T21:31:11.745010332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49022"
level=debug ts=2026-05-13T21:31:24.200864969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:31:24.238427295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:31:54.24077586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:31:58.75822541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:32:06.411934626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:45154"
level=error ts=2026-05-13T21:32:09.480757023Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:32:09.482517893Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:32:09.484206102Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:32:09.484228158Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:32:11.748601413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:54356"
ts=2026-05-13T21:32:24.242541826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:32:48.759793479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:32:54.244624177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:33:06.415844231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:56648"
ts=2026-05-13T21:33:24.246132956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:33:29.158396397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:33:36.417698013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:41394"
ts=2026-05-13T21:33:37.446902433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58238"
ts=2026-05-13T21:33:41.753832887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:47398"
ts=2026-05-13T21:33:54.248677218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:34:13.356764012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:34:13.358605135Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:34:24.251192396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:34:41.756822099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:54708"
ts=2026-05-13T21:34:54.252715709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:35:05.692380823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:35:06.422817154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:42972"
ts=2026-05-13T21:35:24.255187277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:35:43.615488844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:35:54.257271303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:36:06.42585635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:37500"
ts=2026-05-13T21:36:24.259467055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:36:42.497499601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:36:54.26095563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:37:06.429027614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:38460"
ts=2026-05-13T21:37:24.263151978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:37:39.423354633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:37:54.264847033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:38:06.432599785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:56490"
ts=2026-05-13T21:38:07.460339391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33492"
ts=2026-05-13T21:38:11.767666664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:33162"
ts=2026-05-13T21:38:24.267028539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:38:32.463256198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:38:32.464720265Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:38:54.268810768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:39:07.463949555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60190"
ts=2026-05-13T21:39:11.771319361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59336"
ts=2026-05-13T21:39:24.271119885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:39:24.797077066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:39:41.773159688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52014"
ts=2026-05-13T21:39:54.272612001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:40:07.466827394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47932"
level=debug ts=2026-05-13T21:40:17.3946021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:40:24.274308339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:40:37.468589001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34840"
ts=2026-05-13T21:40:54.276520794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:41:02.468383586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:41:06.441454443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:36754"
ts=2026-05-13T21:41:07.470309478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:50936"
ts=2026-05-13T21:41:11.777840893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:54200"
ts=2026-05-13T21:41:24.278917753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:41:52.922495669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:41:54.280523105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:42:06.444783012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:58534"
ts=2026-05-13T21:42:07.473792816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60044"
level=error ts=2026-05-13T21:42:09.482036429Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:42:09.483803582Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:42:09.485607869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:42:09.485634358Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:42:11.781623844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:44644"
ts=2026-05-13T21:42:24.282505684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:42:41.784211871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43746"
level=debug ts=2026-05-13T21:42:43.113038782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:42:43.114608825Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:42:54.284253972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:43:07.477756348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:50390"
ts=2026-05-13T21:43:24.285771615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:43:24.962158062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:43:36.449893128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:46680"
ts=2026-05-13T21:43:54.288158851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:44:07.481141802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43132"
level=debug ts=2026-05-13T21:44:22.312192387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:44:24.290245273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:44:41.790068227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36904"
ts=2026-05-13T21:44:54.292240895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:45:21.509075459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:45:24.293748831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:45:54.295345172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:46:06.456359395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:37522"
ts=2026-05-13T21:46:07.487467783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48514"
level=debug ts=2026-05-13T21:46:10.529022261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:46:24.296823362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:46:36.458825035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55932"
level=debug ts=2026-05-13T21:46:47.536149695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:46:47.538068751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:46:54.299278493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:47:24.301455345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:47:25.232556309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:47:36.461840608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:49020"
ts=2026-05-13T21:47:54.303186709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:48:06.46393274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:54644"
ts=2026-05-13T21:48:11.800734603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:44846"
ts=2026-05-13T21:48:24.305428617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:48:26.208273234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:48:41.802980283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43184"
ts=2026-05-13T21:48:54.307817097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:49:04.943112256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:49:07.496679427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35522"
ts=2026-05-13T21:49:11.804927649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:41490"
ts=2026-05-13T21:49:24.309658003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:49:36.46876777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:45890"
ts=2026-05-13T21:49:41.806764309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:34476"
level=debug ts=2026-05-13T21:49:51.161818896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:49:54.31166781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:50:07.50060809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57774"
ts=2026-05-13T21:50:24.313510274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:50:26.838510903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:50:26.839949962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:50:36.473307562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:40914"
ts=2026-05-13T21:50:41.809688466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:34760"
ts=2026-05-13T21:50:54.315129492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:51:09.825203072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:51:11.811500994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:60998"
ts=2026-05-13T21:51:24.317029375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:51:37.505154851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33326"
ts=2026-05-13T21:51:41.813642419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36962"
ts=2026-05-13T21:51:54.31852749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T21:52:01.977797465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:52:09.479889201Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:52:09.4817443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:52:09.483537906Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T21:52:09.483564805Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:52:24.320624593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:52:36.481042157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:49342"
ts=2026-05-13T21:52:37.51054533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43770"
ts=2026-05-13T21:52:54.322831336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T21:53:02.182833973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:53:06.48342893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:59470"
ts=2026-05-13T21:53:07.512569061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57450"
ts=2026-05-13T21:53:24.325168615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:53:41.820998815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36642"
level=debug ts=2026-05-13T21:53:50.67308507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:53:54.327354557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:54:24.329688359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:54:41.823694309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49802"
level=debug ts=2026-05-13T21:54:49.741170658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:54:49.742971133Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:54:54.33175736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:55:06.490120831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:47850"
ts=2026-05-13T21:55:07.518110882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:37762"
ts=2026-05-13T21:55:24.334332521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:55:37.520352567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:50078"
ts=2026-05-13T21:55:41.826751028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:45504"
level=debug ts=2026-05-13T21:55:51.678292629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:55:54.335903326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:56:07.522276708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34454"
ts=2026-05-13T21:56:11.829244913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40118"
ts=2026-05-13T21:56:24.338080292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:56:41.831223982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40138"
level=debug ts=2026-05-13T21:56:46.926695955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:56:54.340208826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:57:07.525863477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57772"
level=debug ts=2026-05-13T21:57:20.996273735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:57:24.341990065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:57:37.528165883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:42404"
ts=2026-05-13T21:57:54.344449578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T21:58:06.499528897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55946"
ts=2026-05-13T21:58:07.530005879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49298"
level=debug ts=2026-05-13T21:58:11.837960826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:58:24.346829125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T21:58:36.501232953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:59584"
level=debug ts=2026-05-13T21:58:54.006115224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:58:54.007916551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T21:58:54.349074927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T21:59:06.503337609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:42978"
ts=2026-05-13T21:59:07.533920888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47566"
ts=2026-05-13T21:59:24.351197043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T21:59:34.677575548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:59:41.840647661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52948"
ts=2026-05-13T21:59:54.352956369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:00:09.097806853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:00:24.354902092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:00:54.356815336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:01:02.142068852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:01:07.539467943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53390"
ts=2026-05-13T22:01:24.359006626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:01:37.542136691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:53220"
level=debug ts=2026-05-13T22:01:45.924678822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:01:54.361456928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=error ts=2026-05-13T22:02:09.480051266Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:02:09.482043803Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:02:09.483901028Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:02:09.483925266Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:02:24.363830423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:02:32.704156703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:02:32.705624378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:02:54.365788954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:03:11.852571853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:34966"
ts=2026-05-13T22:03:24.367351451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:03:27.087572316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:03:54.369357749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:04:07.548558487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49654"
level=debug ts=2026-05-13T22:04:11.731379594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:04:24.370740983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:04:54.37240055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:04:56.456789049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:05:11.858115103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:42744"
ts=2026-05-13T22:05:24.374567927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:05:31.33077792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:05:54.376152774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:06:06.523245721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:48292"
ts=2026-05-13T22:06:11.860759348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:50594"
level=debug ts=2026-05-13T22:06:17.775601891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:06:17.777040521Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:06:24.378240457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:06:41.862965834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:43408"
ts=2026-05-13T22:06:54.380361682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:07:01.646313624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:07:06.526960756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:52350"
ts=2026-05-13T22:07:24.382414992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:07:41.866985235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:47954"
level=debug ts=2026-05-13T22:07:53.552383653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:07:54.384508567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:08:07.560335127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:36744"
ts=2026-05-13T22:08:24.386247723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:08:34.058820792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:08:54.387903086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:09:11.47970985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:09:24.390069149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:09:41.873023725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:52530"
level=debug ts=2026-05-13T22:09:53.881236528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:09:53.882860698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:09:54.391982771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:10:06.536164904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:49718"
ts=2026-05-13T22:10:24.394346942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:10:32.19537038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:10:36.538726503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:57644"
ts=2026-05-13T22:10:41.876207259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59634"
ts=2026-05-13T22:10:54.396818676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:11:11.878181148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40036"
level=debug ts=2026-05-13T22:11:20.963125729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:11:24.399279595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:11:54.401260706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:11:57.859775786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:12:06.543809319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:46080"
level=error ts=2026-05-13T22:12:09.480728652Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:12:09.482630237Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:12:09.484338629Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:12:09.484358818Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:12:11.881394506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:55886"
ts=2026-05-13T22:12:24.403742984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:12:37.42813186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:12:54.405924532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:13:11.885198348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:59348"
ts=2026-05-13T22:13:24.407935933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:13:37.261451035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:13:37.263186318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:13:37.577308028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60146"
ts=2026-05-13T22:13:54.410388756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:14:07.579698287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58278"
level=debug ts=2026-05-13T22:14:20.528348626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:14:24.412593961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:14:54.413990738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:15:06.55377498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:40250"
ts=2026-05-13T22:15:11.89143959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:46420"
level=debug ts=2026-05-13T22:15:22.699170105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:15:24.416407273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:15:41.897592852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:38496"
ts=2026-05-13T22:15:54.418058469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:16:11.903173651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49256"
level=debug ts=2026-05-13T22:16:15.660603943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:16:24.420403304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:16:41.905624115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36636"
ts=2026-05-13T22:16:54.421826105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:16:55.7106763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:17:06.559522263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55648"
ts=2026-05-13T22:17:24.424037893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:17:36.561894398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:34728"
ts=2026-05-13T22:17:54.425610345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:17:57.078082445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:17:57.07985395Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:18:24.427573432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:18:36.564886252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:39556"
ts=2026-05-13T22:18:54.429870777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:18:58.774304318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:19:06.566734685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:59804"
ts=2026-05-13T22:19:11.912963936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:36604"
ts=2026-05-13T22:19:24.432114601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:19:47.566170497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:19:54.434183531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:20:06.570656106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:36084"
ts=2026-05-13T22:20:24.436058921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:20:43.383839641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:20:54.438147729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:21:06.574024876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:47236"
ts=2026-05-13T22:21:24.439810371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=debug ts=2026-05-13T22:21:35.840031861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:21:36.576275948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:36390"
ts=2026-05-13T22:21:41.920251339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:48892"
ts=2026-05-13T22:21:54.441883209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
level=error ts=2026-05-13T22:22:09.480972489Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:22:09.482674738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:22:09.48434081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:22:09.484359601Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:22:11.922206075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:49746"
level=debug ts=2026-05-13T22:22:23.687208351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:22:23.689113195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:22:24.443827346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:22:37.602252292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59248"
ts=2026-05-13T22:22:41.924304952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:42394"
ts=2026-05-13T22:22:54.445793371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:23:06.580887715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:42530"
ts=2026-05-13T22:23:11.926302408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40934"
ts=2026-05-13T22:23:24.448356177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:23:24.538104006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:23:37.605801857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:39584"
ts=2026-05-13T22:23:54.449991579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:24:06.584896059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:55094"
ts=2026-05-13T22:24:07.607423825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45732"
ts=2026-05-13T22:24:11.929507654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:53564"
level=debug ts=2026-05-13T22:24:14.441128022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:24:24.451466189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:24:54.453740684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:25:07.611363158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60840"
level=debug ts=2026-05-13T22:25:16.87141513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:25:24.456356829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:25:36.589330636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:35956"
ts=2026-05-13T22:25:54.458013289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:25:54.520610007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:26:07.614933629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34242"
ts=2026-05-13T22:26:24.460473175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:26:31.481967592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:26:31.48357282Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:26:54.462539408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:27:06.594418857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:59008"
ts=2026-05-13T22:27:07.618132707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:54382"
ts=2026-05-13T22:27:11.945253618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:40342"
level=debug ts=2026-05-13T22:27:15.456754274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:27:24.464180799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:27:37.620601783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44036"
ts=2026-05-13T22:27:54.465892593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:28:07.622242274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57600"
ts=2026-05-13T22:28:11.948185774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:44536"
level=debug ts=2026-05-13T22:28:16.469383392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:28:24.467253139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:28:37.62490726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:42150"
level=debug ts=2026-05-13T22:28:51.683935033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:28:54.468856425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:29:11.951404796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:57158"
ts=2026-05-13T22:29:24.470751617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:29:36.603054485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:41870"
level=debug ts=2026-05-13T22:29:38.015924096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:29:54.47303879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wz8zs-3a718889 10.128.0.62:7946"
ts=2026-05-13T22:30:06.604910671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:42032"
ts=2026-05-13T22:30:11.954542684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:39136"
level=debug ts=2026-05-13T22:30:18.701202282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:30:18.702957544Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:30:24.474806985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:30:54.476855818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
level=debug ts=2026-05-13T22:30:59.303660354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:31:07.633392203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:56648"
ts=2026-05-13T22:31:11.957797256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:39896"
ts=2026-05-13T22:31:24.478666137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:31:36.609899333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:39400"
ts=2026-05-13T22:31:37.635663617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35382"
level=debug ts=2026-05-13T22:31:43.091598878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:31:54.480745952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ckrtn-4eeff809 10.128.0.61:7946"
ts=2026-05-13T22:32:06.611853417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.62:48694"
level=error ts=2026-05-13T22:32:09.481077722Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:32:09.483145866Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:32:09.484982775Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T22:32:09.485003523Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:32:11.96063678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:34128"
ts=2026-05-13T22:32:24.483230837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:32:28.593959867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:32:41.962220688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.61:46794"
ts=2026-05-13T22:32:54.48474582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
ts=2026-05-13T22:33:24.486239989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3b8bd10d 10.128.0.65:7946"
level=debug ts=2026-05-13T22:33:25.98955492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:33:37.64248194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:39992"