level=info ts=2026-05-14T14:53:13.287725536Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-14T14:53:13.287753171Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-14T14:53:13.289129679Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-14T14:53:13.289424973Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T14:53:13.289490413Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=debug ts=2026-05-14T14:53:13.289992434Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T14:53:13.290051716Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=info ts=2026-05-14T14:53:13.290221808Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-0962096c
ts=2026-05-14T14:53:13.290301531Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-14T14:53:13.290317055Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=debug ts=2026-05-14T14:53:13.290336292Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=info ts=2026-05-14T14:53:13.290529886Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-14T14:53:13.296436501Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T14:53:13.296458424Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-14T14:53:13.296482988Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.955893ms
level=error ts=2026-05-14T14:53:13.296496606Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-14T14:53:13.296512206Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-14T14:53:13.299705313Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T14:53:13.299723744Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T14:53:13.307778858Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-14T14:53:13.307781675Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-14T14:53:13.307810893Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-14T14:53:13.307810596Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-14T14:53:13.307829876Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-14T14:53:13.307897479Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-14T14:53:13.307898122Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-14T14:53:13.30791205Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-14T14:53:13.308508625Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-14T14:53:13.308521003Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-14T14:53:13.308592398Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-14T14:53:13.308592421Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-14T14:53:13.308611049Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-14T14:53:13.308757567Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-14T14:53:13.30877854Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-14T14:53:13.308799463Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-14T14:53:14.309525539Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-14T14:53:14.447776598Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-14T14:53:14.447842168Z caller=loki.go:615 msg="Loki started" startup_time=1.226825602s
ts=2026-05-14T14:53:15.090210288Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T14:53:15.090245329Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:53:17.670482121Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T14:53:17.670518181Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T14:53:19.448873543Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-14T14:53:19.448939777Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-14T14:53:24.691745856Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T14:53:24.691770377Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-14T14:53:31.284024077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:53:33.932163497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:40558"
ts=2026-05-14T14:53:34.447952541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48182"
ts=2026-05-14T14:53:36.0250201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.64:7946"
ts=2026-05-14T14:53:36.027163849Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.94:7946: dial tcp 10.128.0.94:7946: connect: connection refused"
ts=2026-05-14T14:53:36.028012756Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.90:7946: dial tcp 10.128.0.90:7946: connect: connection refused"
ts=2026-05-14T14:53:36.028757451Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.92:7946: dial tcp 10.128.0.92:7946: connect: connection refused"
ts=2026-05-14T14:53:36.02883356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.93:7946"
ts=2026-05-14T14:53:36.02887675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49644"
ts=2026-05-14T14:53:36.03015194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
level=info ts=2026-05-14T14:53:36.031278653Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=22.73475937s
ts=2026-05-14T14:53:38.026513028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36910"
level=debug ts=2026-05-14T14:53:45.403411831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:53:51.79634381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T14:53:54.775366652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54948"
level=debug ts=2026-05-14T14:54:03.349815541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:54:21.798535925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T14:54:24.7786097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:59842"
level=debug ts=2026-05-14T14:54:30.467862702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:54:51.800294814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T14:54:57.446879598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T14:54:57.448663821Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T14:55:21.80216539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T14:55:24.783200083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:60896"
level=debug ts=2026-05-14T14:55:30.62401141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:55:51.804639719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T14:56:20.150268136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:56:21.805662562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T14:56:24.788419902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:36122"
ts=2026-05-14T14:56:51.807624576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T14:56:52.256482896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33748"
ts=2026-05-14T14:56:54.791734063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:48086"
ts=2026-05-14T14:57:18.104502324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60186"
ts=2026-05-14T14:57:21.809703908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T14:57:21.864175419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:57:24.795220479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:36186"
ts=2026-05-14T14:57:51.810949496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T14:57:52.262017708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40298"
ts=2026-05-14T14:58:21.812697951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T14:58:22.263644916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:37010"
level=debug ts=2026-05-14T14:58:29.197208986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T14:58:51.814878339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T14:58:52.264887581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40910"
level=debug ts=2026-05-14T14:59:19.945426302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T14:59:19.946936546Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T14:59:21.817578387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T14:59:24.804927623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:34736"
ts=2026-05-14T14:59:48.120779392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52158"
ts=2026-05-14T14:59:51.820890664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:00:21.823189775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:00:21.870124719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:00:24.810150552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38764"
ts=2026-05-14T15:00:51.825181565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:00:54.812821581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54936"
level=debug ts=2026-05-14T15:01:00.416403183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:01:21.828284285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:01:51.831237241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:01:52.280128124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38672"
level=debug ts=2026-05-14T15:02:02.334802153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:02:21.83353123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:02:24.822800039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47570"
level=debug ts=2026-05-14T15:02:45.268647796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:02:51.835621617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:02:54.82620726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41034"
level=info ts=2026-05-14T15:03:19.449653736Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-14T15:03:19.453850316Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:03:19.455940729Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:03:19.457748968Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:03:19.457776302Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T15:03:19.457788492Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-14T15:03:21.838524123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:03:45.7950858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:03:45.796931983Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:03:51.841523408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:04:21.845097255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:04:22.293527817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:56650"
level=debug ts=2026-05-14T15:04:34.077005065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:04:48.149524973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38198"
ts=2026-05-14T15:04:51.84822167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:04:52.29500723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40254"
ts=2026-05-14T15:05:21.851453223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:05:31.655422703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:05:51.85392773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:05:54.843287421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41214"
level=debug ts=2026-05-14T15:06:14.234486807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:06:21.857725928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:06:22.305708309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:47700"
ts=2026-05-14T15:06:51.861141543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:06:54.848380304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:46204"
level=debug ts=2026-05-14T15:07:13.529615688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:07:21.862737673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:07:22.309887906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:47542"
ts=2026-05-14T15:07:51.864518253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:07:52.311138143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36056"
level=debug ts=2026-05-14T15:08:03.599922809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:08:03.601463541Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:08:21.86660494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:08:51.86914607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:08:54.858536554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:55296"
level=debug ts=2026-05-14T15:08:58.6074735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:09:21.871993411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:09:45.719863426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:09:51.875203113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:10:21.876975948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:10:21.896101342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:10:51.879692005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:10:52.328728666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50542"
ts=2026-05-14T15:11:21.882543692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:11:22.233258233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:11:22.330374768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38984"
ts=2026-05-14T15:11:24.872757881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:32952"
ts=2026-05-14T15:11:51.885423808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:11:54.87694836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:46660"
level=debug ts=2026-05-14T15:12:00.348011227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:12:00.350417476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:12:18.194763112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:44546"
ts=2026-05-14T15:12:21.887313622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:12:43.99703052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:12:51.890207638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=error ts=2026-05-14T15:13:19.462824837Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:13:19.465319806Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:13:19.467532935Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:13:19.467563697Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:13:21.893366446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:13:43.504278009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:13:51.895633554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:13:54.889238136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54608"
level=debug ts=2026-05-14T15:14:21.858525982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:14:21.89830918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:14:51.902511894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:14:54.895011173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58586"
level=debug ts=2026-05-14T15:15:02.469771566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:15:21.904998793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:15:37.19439565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:15:37.19643082Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:15:51.907341189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:16:12.455523834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:16:18.214659746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35624"
ts=2026-05-14T15:16:21.909710734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:16:24.90187982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:60096"
ts=2026-05-14T15:16:51.91234693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:16:52.367608645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59704"
level=debug ts=2026-05-14T15:17:04.496703795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:17:18.219975279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35380"
ts=2026-05-14T15:17:21.915545433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:17:48.222602274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59558"
ts=2026-05-14T15:17:51.917514594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:17:52.37218895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33890"
level=debug ts=2026-05-14T15:17:52.485016947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:18:21.92028979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:18:22.374019422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:60948"
level=debug ts=2026-05-14T15:18:39.90861481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:18:48.22828961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57574"
ts=2026-05-14T15:18:51.922487615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:19:18.231720513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36164"
ts=2026-05-14T15:19:21.925123275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:19:24.918692518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41236"
level=debug ts=2026-05-14T15:19:33.575771346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:19:33.577636191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:19:51.927893876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:20:11.017511915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:20:21.931029589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:20:24.923667926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:40610"
ts=2026-05-14T15:20:51.934484933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:20:52.388845502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36112"
level=debug ts=2026-05-14T15:20:53.528835937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:21:21.937050782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:21:30.313411113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:21:51.939112154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:22:18.24849404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51300"
ts=2026-05-14T15:22:21.940940255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:22:22.395954626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:46570"
level=debug ts=2026-05-14T15:22:28.231814529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:22:48.251773343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50930"
ts=2026-05-14T15:22:51.943726356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:22:54.936161937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:60154"
level=debug ts=2026-05-14T15:23:09.952172424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:23:09.954236535Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:23:18.254556824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37630"
level=error ts=2026-05-14T15:23:19.461109148Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:23:19.46313071Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:23:19.465136183Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:23:19.465159786Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:23:21.945494269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:23:22.400472912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:56854"
level=debug ts=2026-05-14T15:23:46.686494848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:23:51.948128971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:23:52.402128814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48564"
ts=2026-05-14T15:24:21.951074456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:24:22.403338401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59666"
level=debug ts=2026-05-14T15:24:33.303137858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:24:51.954316955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:24:52.404583763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:43718"
ts=2026-05-14T15:25:18.266885547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52920"
ts=2026-05-14T15:25:21.957616536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:25:23.57727176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:25:48.270204336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49074"
ts=2026-05-14T15:25:51.960012465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:25:54.9496307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:35044"
ts=2026-05-14T15:26:21.961727337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:26:23.299211607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:26:51.963970957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:26:52.415857019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57426"
ts=2026-05-14T15:26:54.955295105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:33004"
level=debug ts=2026-05-14T15:26:58.729529814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:26:58.731576834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:27:18.277842353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36524"
ts=2026-05-14T15:27:21.96751512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:27:48.281306625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59514"
ts=2026-05-14T15:27:51.970815102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:27:52.420224113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59346"
level=debug ts=2026-05-14T15:27:54.8498071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:27:54.962686654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:53660"
ts=2026-05-14T15:28:21.973079888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:28:48.286887214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37772"
level=debug ts=2026-05-14T15:28:50.689035185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:28:51.975736214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:28:52.423887329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33388"
ts=2026-05-14T15:29:21.978921363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:29:24.971653583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:37528"
level=debug ts=2026-05-14T15:29:33.713108282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:29:51.982423586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:29:54.974600224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:43240"
level=debug ts=2026-05-14T15:30:12.130247418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:30:21.984601269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:30:22.431614114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57150"
ts=2026-05-14T15:30:48.295576432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51328"
ts=2026-05-14T15:30:51.985566654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:30:54.980585222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47238"
level=debug ts=2026-05-14T15:31:01.937473796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:31:01.939018222Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:31:21.98722649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:31:22.435630865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36834"
ts=2026-05-14T15:31:24.983209798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54370"
level=debug ts=2026-05-14T15:31:45.574613319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:31:51.989315862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:31:54.986097433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54514"
ts=2026-05-14T15:32:21.991912992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:32:22.44139042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57710"
ts=2026-05-14T15:32:24.989185559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:44490"
level=debug ts=2026-05-14T15:32:30.98321431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:32:51.993113326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:32:52.443060931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33682"
level=debug ts=2026-05-14T15:33:06.078171452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:33:19.461703445Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:33:19.463752354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:33:19.465664601Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:33:19.465688347Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:33:21.995471719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:33:22.445096731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41736"
ts=2026-05-14T15:33:24.995765471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54462"
ts=2026-05-14T15:33:51.997397138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:34:01.97394896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:34:18.312101629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45322"
ts=2026-05-14T15:34:21.999181056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:34:22.450703575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48630"
ts=2026-05-14T15:34:52.001669694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:34:59.793619602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:34:59.795416911Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:35:22.00473288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:35:25.008919078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58632"
level=debug ts=2026-05-14T15:35:43.879096725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:35:48.320369075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45242"
ts=2026-05-14T15:35:52.008283619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:35:55.012325377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58390"
ts=2026-05-14T15:36:22.011545326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:36:22.466405589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41946"
level=debug ts=2026-05-14T15:36:24.593291916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:36:52.014689093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:37:02.637903851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:37:22.017847657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:37:52.020564778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:37:52.473207189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38100"
ts=2026-05-14T15:37:55.022824587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:39348"
level=debug ts=2026-05-14T15:37:59.212584784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:38:22.022686642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:38:49.728086931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:38:49.730197952Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:38:52.023776352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:39:22.025490637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:39:30.798030399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:39:52.028584888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:39:55.033597444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:52636"
level=debug ts=2026-05-14T15:40:10.656715002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:40:22.030231276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:40:22.487089186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50764"
ts=2026-05-14T15:40:52.032558474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:40:54.180371028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:40:55.038962213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:52614"
ts=2026-05-14T15:41:22.034431936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:41:22.491460105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38404"
ts=2026-05-14T15:41:25.041974058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:36440"
level=debug ts=2026-05-14T15:41:39.003588863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:41:52.035825327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:42:18.355286709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45220"
level=debug ts=2026-05-14T15:42:21.848579366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:42:21.850631102Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:42:22.038276843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:42:48.358186097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45402"
ts=2026-05-14T15:42:52.041158477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:42:55.049246695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:50088"
level=error ts=2026-05-14T15:43:19.461418241Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:43:19.46342684Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:43:19.465166156Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:43:19.465191291Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:43:22.044395127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:43:22.169153042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:43:22.501834243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:47502"
ts=2026-05-14T15:43:52.046233588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:43:55.055034245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:34620"
level=debug ts=2026-05-14T15:44:08.993855549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:44:22.048254724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:44:22.506756653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:53844"
ts=2026-05-14T15:44:52.050901707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:44:57.868887336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:45:22.053207762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:45:35.032088756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:45:48.370442934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43858"
ts=2026-05-14T15:45:52.055676434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:46:12.92567739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:46:12.927474788Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:46:22.058518452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:46:22.517670061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51798"
ts=2026-05-14T15:46:25.068287065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:44046"
ts=2026-05-14T15:46:52.061387507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:47:03.282012157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:47:18.37908443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:47042"
ts=2026-05-14T15:47:22.063750863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:47:22.522175453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57008"
ts=2026-05-14T15:47:25.074239581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38384"
ts=2026-05-14T15:47:52.066073167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:47:54.882516946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:48:22.068597915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:48:22.526879981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:54262"
ts=2026-05-14T15:48:25.080669482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:36468"
level=debug ts=2026-05-14T15:48:34.153145113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:48:52.070140733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:49:15.870794085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:49:22.071947483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:49:25.08643314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:37228"
ts=2026-05-14T15:49:48.391548024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60864"
ts=2026-05-14T15:49:52.075517797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:49:57.462653166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:49:57.464327902Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:50:22.078372873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:50:25.09111653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:60376"
ts=2026-05-14T15:50:52.08134806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:50:52.993477518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:51:22.084638658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:51:34.887743123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:51:52.086623808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:51:52.547389985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41430"
ts=2026-05-14T15:52:18.405709515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53632"
ts=2026-05-14T15:52:22.088538372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:52:25.10190141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:59050"
level=debug ts=2026-05-14T15:52:33.540280017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:52:52.091168241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:52:52.551612431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51868"
level=debug ts=2026-05-14T15:53:13.865471794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:53:19.462814161Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:53:19.464809638Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:53:19.467422285Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T15:53:19.467447718Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:53:22.0933723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:53:25.108413226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:43308"
ts=2026-05-14T15:53:52.095853374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T15:53:52.133887785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:53:52.135477672Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:54:22.099215745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T15:54:42.477861623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:54:52.102249982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:54:55.115425084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58072"
ts=2026-05-14T15:55:22.105631779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:55:22.566762217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:39848"
ts=2026-05-14T15:55:25.118330995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:46462"
level=debug ts=2026-05-14T15:55:33.703722845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:55:52.107479749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T15:56:13.676060783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:56:22.109931294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:56:22.571186838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:42774"
ts=2026-05-14T15:56:25.124790721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47266"
ts=2026-05-14T15:56:48.427341751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46842"
ts=2026-05-14T15:56:52.112874566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:56:55.128006847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:53510"
level=debug ts=2026-05-14T15:57:09.204220589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:57:18.430599216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50316"
ts=2026-05-14T15:57:22.116337467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T15:57:22.57512148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:39652"
ts=2026-05-14T15:57:25.131373961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58264"
level=debug ts=2026-05-14T15:57:50.109630206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T15:57:50.111499223Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T15:57:52.119747885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:57:52.576052486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:60418"
ts=2026-05-14T15:57:55.13429789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:50212"
ts=2026-05-14T15:58:18.43596333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51010"
ts=2026-05-14T15:58:22.121763885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:58:25.137936857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:49184"
level=debug ts=2026-05-14T15:58:49.307296647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:58:52.123602384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T15:59:18.441047625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33448"
ts=2026-05-14T15:59:22.126447963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T15:59:25.144625987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41000"
level=debug ts=2026-05-14T15:59:35.195380962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T15:59:52.129481179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:00:22.132292898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T16:00:25.839808655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:00:52.135803169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:00:52.592978056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:35782"
level=debug ts=2026-05-14T16:01:20.452714968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:01:22.139211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:01:22.594822646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:46694"
ts=2026-05-14T16:01:25.155917967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:50408"
ts=2026-05-14T16:01:52.141967104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:01:52.595994107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57220"
ts=2026-05-14T16:01:55.159333687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:54056"
level=debug ts=2026-05-14T16:02:06.366086065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:02:06.367999977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:02:22.144200774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:02:22.597389462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51656"
level=debug ts=2026-05-14T16:02:43.763778338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:02:52.147012634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:02:52.598722909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41870"
ts=2026-05-14T16:02:55.165245847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:57922"
level=error ts=2026-05-14T16:03:19.462258048Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:03:19.46416655Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:03:19.466003301Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:03:19.466031767Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:03:22.149830841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:03:25.168123459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:48560"
level=debug ts=2026-05-14T16:03:25.346635959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:03:48.465530001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48892"
ts=2026-05-14T16:03:52.151805487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:04:03.879875861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:04:22.153203084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:04:38.628321403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:04:52.155243381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:05:22.158278881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:05:22.611147982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40736"
ts=2026-05-14T16:05:25.178578632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:45622"
level=debug ts=2026-05-14T16:05:40.003074816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:05:40.004701033Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:05:52.160819932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:05:52.612304286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41698"
ts=2026-05-14T16:05:55.181762472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:56838"
ts=2026-05-14T16:06:18.477897951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59840"
level=debug ts=2026-05-14T16:06:19.286426103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:06:22.162839331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:06:25.185254198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:43284"
ts=2026-05-14T16:06:48.481117938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35614"
ts=2026-05-14T16:06:52.165795711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T16:07:09.538594099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:07:22.168557167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:07:48.486748801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38812"
ts=2026-05-14T16:07:52.170995293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:08:00.287887059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:08:22.172125743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:08:22.625167506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48988"
ts=2026-05-14T16:08:25.194910058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:57408"
ts=2026-05-14T16:08:52.174419378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:08:52.626502595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38134"
level=debug ts=2026-05-14T16:09:00.214305836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:09:18.494153078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35658"
ts=2026-05-14T16:09:22.177286373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:09:22.628035301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48126"
ts=2026-05-14T16:09:48.49648477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51944"
ts=2026-05-14T16:09:52.178880503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:09:52.629879712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:58840"
ts=2026-05-14T16:09:55.202792955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47894"
level=debug ts=2026-05-14T16:10:00.658627274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:10:00.66025886Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:10:22.18061819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:10:48.501761273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49438"
level=debug ts=2026-05-14T16:10:49.611065199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:10:52.183121798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:10:52.634601146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:47512"
ts=2026-05-14T16:10:55.208582162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:39650"
ts=2026-05-14T16:11:18.504768137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33006"
ts=2026-05-14T16:11:22.185411078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T16:11:46.199332425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:11:52.18724544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:11:52.638685552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33730"
ts=2026-05-14T16:12:22.189922222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T16:12:47.96597887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:12:52.193169496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:13:18.51526848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43078"
level=error ts=2026-05-14T16:13:19.461842235Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:13:19.46359677Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:13:19.465167307Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:13:19.465192207Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:13:22.196072648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T16:13:32.576689403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:13:52.198494371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:13:55.223867646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:52640"
ts=2026-05-14T16:14:18.519991271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45934"
level=debug ts=2026-05-14T16:14:18.81918362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:14:18.820967284Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:14:22.201423927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:14:22.653670381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52420"
ts=2026-05-14T16:14:25.226749355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47264"
ts=2026-05-14T16:14:52.204385979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T16:15:07.769512225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:15:22.206102326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:15:44.071805881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:15:52.207134595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:16:21.605281959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:16:22.209525669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:16:48.534916761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40418"
ts=2026-05-14T16:16:52.212746298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T16:17:11.004127713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:17:22.216009862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:17:22.671223417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:35658"
ts=2026-05-14T16:17:52.218567331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:18:03.826106537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:18:03.827561807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:18:18.542938425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48244"
ts=2026-05-14T16:18:22.219761356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:18:22.675301906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36800"
ts=2026-05-14T16:18:52.222434342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:18:55.24890906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:48172"
level=debug ts=2026-05-14T16:19:03.36873368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:19:22.224953657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:19:25.251274363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:53148"
ts=2026-05-14T16:19:52.226428708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
level=debug ts=2026-05-14T16:19:59.074453647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:20:22.228977206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:20:25.257177063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:49360"
level=debug ts=2026-05-14T16:20:48.145832156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:20:48.557013876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:32816"
ts=2026-05-14T16:20:52.23088255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:20:52.688716392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50774"
ts=2026-05-14T16:21:18.559753429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35820"
ts=2026-05-14T16:21:22.233512834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T16:21:47.127667494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:21:52.23516553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:21:52.69382771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59790"
ts=2026-05-14T16:22:22.237013953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
level=debug ts=2026-05-14T16:22:47.995087211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:22:47.996785303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:22:48.567970211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58054"
ts=2026-05-14T16:22:52.239974279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:22:52.697772432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33452"
level=error ts=2026-05-14T16:23:19.464931866Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:23:19.466963419Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:23:19.468792333Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T16:23:19.468822755Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:23:22.242166676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5c9c3e65 10.128.0.91:7946"
ts=2026-05-14T16:23:22.699143317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50700"
level=debug ts=2026-05-14T16:23:48.281722886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:23:52.244086461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:23:52.700735126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:34766"
ts=2026-05-14T16:24:22.247526411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:24:25.277383488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58254"
level=debug ts=2026-05-14T16:24:27.504510158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:24:48.578412703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59258"
ts=2026-05-14T16:24:52.249984129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
level=debug ts=2026-05-14T16:25:22.018505649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:25:22.252630625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:25:22.708408634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:49894"
ts=2026-05-14T16:25:25.283926195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38122"
ts=2026-05-14T16:25:52.255951979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:25:55.287374715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:40888"
level=debug ts=2026-05-14T16:26:15.357635016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:26:22.259107106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:26:25.290623167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:44876"
ts=2026-05-14T16:26:52.261795028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-np5w2-4bfe15a2 10.129.0.64:7946"
ts=2026-05-14T16:26:55.293518164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:42866"
level=debug ts=2026-05-14T16:27:03.155781146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T16:27:03.157418492Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T16:27:22.264675231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:27:22.718340057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44368"
ts=2026-05-14T16:27:25.296573606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:37822"
level=debug ts=2026-05-14T16:27:47.334891153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T16:27:52.26809624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:28:18.597312638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52786"
ts=2026-05-14T16:28:22.270987825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-htbdg-77579175 10.130.0.80:7946"
ts=2026-05-14T16:28:25.302315514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:33882"
level=debug ts=2026-05-14T16:28:40.79357779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"