level=info ts=2026-05-11T23:34:31.598765346Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-11T23:34:31.598798071Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-11T23:34:31.600200763Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-11T23:34:31.600613174Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T23:34:31.60071066Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.87 inf=eth0
level=debug ts=2026-05-11T23:34:31.60122704Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T23:34:31.601314983Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.87 inf=eth0
level=info ts=2026-05-11T23:34:31.601398994Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-611cb595
ts=2026-05-11T23:34:31.601489887Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-11T23:34:31.601501684Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.87 advertisePort=7946
level=debug ts=2026-05-11T23:34:31.601517904Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.87 advertisePort=7946
level=info ts=2026-05-11T23:34:31.60164499Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-11T23:34:31.606459823Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T23:34:31.606475732Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T23:34:31.606485572Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.842754ms
level=error ts=2026-05-11T23:34:31.606493729Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-11T23:34:31.606505033Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-11T23:34:31.609099175Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T23:34:31.609117006Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T23:34:31.61528005Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-11T23:34:31.615300749Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-11T23:34:31.6153441Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-11T23:34:31.615356667Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-11T23:34:31.615380076Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-11T23:34:31.615396783Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-11T23:34:31.615461342Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-11T23:34:31.615468047Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-11T23:34:31.615529471Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-11T23:34:31.615537687Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-11T23:34:31.615589545Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-11T23:34:31.615756616Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-11T23:34:31.615845856Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-11T23:34:31.616228217Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-11T23:34:31.616243024Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-11T23:34:31.616267892Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-11T23:34:32.616612002Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-11T23:34:32.631348494Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T23:34:32.631388255Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T23:34:32.735616753Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-11T23:34:32.735660785Z caller=loki.go:615 msg="Loki started" startup_time=1.20311001s
ts=2026-05-11T23:34:34.868616054Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T23:34:34.86863892Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-11T23:34:37.735984857Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-11T23:34:37.736051091Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-11T23:34:42.46140914Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T23:34:42.461449631Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T23:34:47.422535117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:34:51.829657693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53470"
ts=2026-05-11T23:34:54.311410651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48060"
ts=2026-05-11T23:34:56.053407549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-05-11T23:34:56.054610629Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.86:7946: dial tcp 10.130.0.86:7946: connect: connection refused"
ts=2026-05-11T23:34:56.054743343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.79:7946"
ts=2026-05-11T23:34:56.055621582Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.84:7946: dial tcp 10.130.0.84:7946: connect: connection refused"
ts=2026-05-11T23:34:56.056455216Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.85:7946: dial tcp 10.130.0.85:7946: connect: connection refused"
ts=2026-05-11T23:34:56.056538125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.87:7946"
ts=2026-05-11T23:34:56.056564449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:34816"
level=info ts=2026-05-11T23:34:56.056769656Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.450260634s
level=debug ts=2026-05-11T23:35:07.149666212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:35:08.241253823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:35:09.609884859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52722"
ts=2026-05-11T23:35:13.44518045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57510"
level=debug ts=2026-05-11T23:35:28.119612345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:35:38.242550691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-11T23:35:49.938204594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:35:51.28849645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58772"
ts=2026-05-11T23:36:00.01284553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35856"
ts=2026-05-11T23:36:08.244877541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-11T23:36:25.528600098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T23:36:25.53014127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:36:38.247625136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:37:00.018752089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46846"
level=debug ts=2026-05-11T23:37:06.893726357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:37:08.252647013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:37:38.256008279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:37:51.295994375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47628"
ts=2026-05-11T23:38:08.257670847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-11T23:38:17.013713295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:38:38.259953046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:39:00.030217227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59826"
ts=2026-05-11T23:39:08.262713362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:39:21.301471351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39130"
level=debug ts=2026-05-11T23:39:29.688128351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:39:38.265056533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:40:08.266426012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:40:13.468644165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42572"
ts=2026-05-11T23:40:21.305000917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51234"
level=debug ts=2026-05-11T23:40:29.501757348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T23:40:38.267352828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:41:08.26866996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:41:13.472702852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42696"
ts=2026-05-11T23:41:30.045139758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58742"
level=debug ts=2026-05-11T23:41:34.986504854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T23:41:34.987924982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:41:38.270400787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:42:08.272424209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:42:13.476759851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34542"
level=debug ts=2026-05-11T23:42:37.470409717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:42:38.27446325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:43:08.275470106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-11T23:43:33.708981458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:43:38.277792121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:43:51.316854834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33778"
ts=2026-05-11T23:44:00.060625393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53218"
ts=2026-05-11T23:44:08.280663687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:44:13.485217088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34480"
level=debug ts=2026-05-11T23:44:25.819389172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T23:44:37.736166178Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-11T23:44:37.741843469Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:44:37.743746703Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:44:37.74552693Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:44:37.745553261Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T23:44:37.745568445Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-11T23:44:38.283691189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:44:51.320908577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35980"
ts=2026-05-11T23:45:00.066591926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51034"
level=debug ts=2026-05-11T23:45:01.912026789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:45:08.285823598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:45:21.321708313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52744"
ts=2026-05-11T23:45:38.287709307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:45:51.32278496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41218"
level=debug ts=2026-05-11T23:45:53.928903045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T23:45:53.930949331Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:46:08.289803459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:46:38.29319883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-11T23:46:51.246579371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:47:00.079208891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37632"
ts=2026-05-11T23:47:08.294921862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:47:13.496846512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35612"
ts=2026-05-11T23:47:21.330889607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56424"
level=debug ts=2026-05-11T23:47:31.187516066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:47:38.296777183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:47:51.332161368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56568"
ts=2026-05-11T23:48:08.298560694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:48:13.50139144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51970"
level=debug ts=2026-05-11T23:48:20.558995455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:48:30.087854863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48318"
ts=2026-05-11T23:48:38.300582655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:49:00.091513389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53806"
level=debug ts=2026-05-11T23:49:02.250632459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:49:08.301616493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:49:38.303113767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:49:43.506838643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52354"
level=debug ts=2026-05-11T23:50:03.616791964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T23:50:03.618885768Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:50:08.303817537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:50:30.100566188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50016"
ts=2026-05-11T23:50:38.305202523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:50:43.511383207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35298"
level=debug ts=2026-05-11T23:50:43.903685603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:50:51.347400738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59034"
ts=2026-05-11T23:51:00.103683125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48978"
ts=2026-05-11T23:51:08.306801819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:51:30.106768116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39328"
ts=2026-05-11T23:51:38.307702325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-11T23:51:38.601894935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:52:00.109611652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40384"
ts=2026-05-11T23:52:08.309418677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-11T23:52:21.611888966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:52:38.311692102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:52:51.354511951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59874"
ts=2026-05-11T23:53:08.314016516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:53:21.356301961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58932"
level=debug ts=2026-05-11T23:53:21.554879253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:53:38.316629514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-11T23:53:43.525732797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53946"
ts=2026-05-11T23:53:51.357729594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57932"
ts=2026-05-11T23:54:08.319099148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:54:13.527113343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38748"
level=debug ts=2026-05-11T23:54:17.660136061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T23:54:17.662273842Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:54:21.359272785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46170"
ts=2026-05-11T23:54:30.124272118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:32780"
level=error ts=2026-05-11T23:54:37.749489862Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:54:37.751391539Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:54:37.75321055Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-11T23:54:37.753230139Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:54:38.321788186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-11T23:55:04.45147654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:55:08.324479864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:55:38.325303769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-11T23:55:45.211096245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:55:51.364487606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59866"
ts=2026-05-11T23:56:00.135741616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34704"
ts=2026-05-11T23:56:08.327169075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:56:38.329649685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-11T23:56:39.907534874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:57:08.332621791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-11T23:57:15.520703727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:57:21.369141933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42974"
ts=2026-05-11T23:57:38.335016828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-11T23:57:59.115076569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T23:57:59.117146002Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-11T23:58:00.158641236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58474"
ts=2026-05-11T23:58:08.336631615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:58:30.162267566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49952"
ts=2026-05-11T23:58:38.338034041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-11T23:58:43.552082152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60352"
level=debug ts=2026-05-11T23:58:54.5219522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:59:08.344796387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-11T23:59:13.553602635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60464"
ts=2026-05-11T23:59:30.16823915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48150"
level=debug ts=2026-05-11T23:59:31.832524039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T23:59:38.347033721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:00:00.176194692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42358"
ts=2026-05-12T00:00:08.348719891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:00:19.263054291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:00:21.382866849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47322"
ts=2026-05-12T00:00:38.351298407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:00:51.3845497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45884"
ts=2026-05-12T00:01:08.354748302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:01:13.677366163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:01:38.358211888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:02:02.443687464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:02:02.445475101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:02:08.360270361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:02:21.390225796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47792"
ts=2026-05-12T00:02:30.193283218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42400"
ts=2026-05-12T00:02:38.362264451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:02:43.566853075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34404"
level=debug ts=2026-05-12T00:02:59.799282435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:03:08.363171138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:03:13.568285048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58398"
ts=2026-05-12T00:03:38.364140397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:03:59.731916659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:04:00.203784975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34382"
ts=2026-05-12T00:04:08.365703558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=error ts=2026-05-12T00:04:37.750013544Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:04:37.752128394Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:04:37.754002117Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:04:37.754032529Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:04:38.36832019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:04:51.040991783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:04:51.399463961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34278"
ts=2026-05-12T00:05:08.371123045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:05:13.574758873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49098"
ts=2026-05-12T00:05:38.37382519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:05:46.354540359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:06:08.377075908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:06:21.40509378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45042"
ts=2026-05-12T00:06:30.220284206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35586"
level=debug ts=2026-05-12T00:06:34.634190004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:06:34.635811693Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:06:38.379494053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:06:43.581771385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54234"
ts=2026-05-12T00:07:08.381541899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:07:32.158185858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:07:38.383973049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:08:00.229252205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56264"
ts=2026-05-12T00:08:08.385733745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:08:15.67372302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:08:21.417786226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56864"
ts=2026-05-12T00:08:38.386956203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:08:49.85108031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:09:00.235735419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39828"
ts=2026-05-12T00:09:08.388418263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:09:31.555511511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:09:38.389689712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:10:00.241504181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44818"
ts=2026-05-12T00:10:08.391939634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:10:13.59329011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:47844"
level=debug ts=2026-05-12T00:10:15.26446115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:10:15.26650539Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:10:21.425028648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35670"
ts=2026-05-12T00:10:38.393727812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:10:43.594863352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60606"
ts=2026-05-12T00:10:51.42684614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55566"
level=debug ts=2026-05-12T00:10:54.411259071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:11:08.396036951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:11:36.554003465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:11:38.397676058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:11:43.599312581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42338"
ts=2026-05-12T00:12:08.400297093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:12:15.022909923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:12:38.402855238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:12:43.602134194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53820"
ts=2026-05-12T00:13:00.259812305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41480"
level=debug ts=2026-05-12T00:13:01.469674474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:13:08.405122966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:13:13.603103519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52412"
ts=2026-05-12T00:13:38.407870912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:14:02.547781012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:14:02.550527876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:14:08.409537257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:14:13.612062874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60830"
level=error ts=2026-05-12T00:14:37.750105505Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:14:37.752305472Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:14:37.754366015Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:14:37.754392394Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:14:38.412298094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:14:41.625280154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:14:43.613365995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34958"
ts=2026-05-12T00:15:00.27215158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38670"
ts=2026-05-12T00:15:08.415484994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:15:18.215971388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:15:38.418889184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:15:43.615099772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45798"
ts=2026-05-12T00:16:08.422034059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:16:17.195390062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:16:38.424625132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:16:43.616998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:44636"
ts=2026-05-12T00:17:08.426108427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:17:10.690589231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:17:13.618534849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52478"
ts=2026-05-12T00:17:21.460524514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44060"
ts=2026-05-12T00:17:38.427886038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:17:51.462433219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58770"
level=debug ts=2026-05-12T00:18:07.296979293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:18:07.298616243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:18:08.429026957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:18:13.620617297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33040"
ts=2026-05-12T00:18:38.430128707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:18:43.622502268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34850"
ts=2026-05-12T00:18:51.46722178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33412"
ts=2026-05-12T00:19:00.296702944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33668"
level=debug ts=2026-05-12T00:19:04.745072571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:19:08.432776397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:19:13.623428446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58584"
ts=2026-05-12T00:19:38.435335772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:19:51.471938885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42556"
level=debug ts=2026-05-12T00:19:55.200806388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:20:00.302546038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48874"
ts=2026-05-12T00:20:08.436979792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:20:33.239507622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:20:38.438698585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:20:51.476375651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42146"
ts=2026-05-12T00:21:00.308193551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36754"
ts=2026-05-12T00:21:08.439785161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:21:13.633418719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49890"
level=debug ts=2026-05-12T00:21:19.917776839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:21:21.477576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53046"
ts=2026-05-12T00:21:38.44053791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:22:08.441325716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:22:17.21796547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:22:17.219977105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:22:21.482294281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41840"
ts=2026-05-12T00:22:30.31772273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47920"
ts=2026-05-12T00:22:38.443143575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:22:51.957139108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:23:08.444631644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:23:38.445741464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:23:41.556618313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:23:51.486744586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47872"
ts=2026-05-12T00:24:08.447993843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:24:16.967486891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:24:37.749215347Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:24:37.751381456Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:24:37.753244378Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:24:37.753269353Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:24:38.451145811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:25:00.333624096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46992"
ts=2026-05-12T00:25:08.452419857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:25:10.610298962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:25:21.492053388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50120"
ts=2026-05-12T00:25:38.454377165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:25:51.493708023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43710"
level=debug ts=2026-05-12T00:26:03.25099545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:26:03.252882476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:26:08.455956441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:26:38.457734064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:26:43.66212574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54128"
level=debug ts=2026-05-12T00:26:50.731313726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:27:00.34621213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59770"
ts=2026-05-12T00:27:08.458759171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:27:13.663045185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55006"
level=debug ts=2026-05-12T00:27:30.083936912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:27:38.461152855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:28:00.351632213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41836"
ts=2026-05-12T00:28:08.463702869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:28:18.25502309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:28:38.466842229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:28:51.507975737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47636"
level=debug ts=2026-05-12T00:28:53.592907027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:29:08.470318221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:29:13.672250799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38300"
ts=2026-05-12T00:29:30.361753544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50884"
ts=2026-05-12T00:29:38.472835479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:29:47.606804491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:29:47.611973954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:29:51.511115552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48702"
ts=2026-05-12T00:30:00.365034803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60454"
ts=2026-05-12T00:30:08.473723546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:30:31.46811594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:30:38.475215121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:30:43.678362439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43298"
ts=2026-05-12T00:31:00.371411776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47300"
ts=2026-05-12T00:31:08.476918307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:31:13.679135639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:56152"
level=debug ts=2026-05-12T00:31:14.116587166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:31:21.515601174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48742"
ts=2026-05-12T00:31:38.478696822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:31:43.680597319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57860"
ts=2026-05-12T00:32:08.479849875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:32:13.682005568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49996"
level=debug ts=2026-05-12T00:32:15.911525988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:32:30.380076237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55912"
ts=2026-05-12T00:32:38.481518213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:32:43.683309295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:39118"
ts=2026-05-12T00:33:08.482965759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:33:11.648591344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:33:38.484683966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:33:55.982252648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:33:55.983995313Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:34:08.486352743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:34:13.6905122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40704"
ts=2026-05-12T00:34:30.391655592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48358"
level=error ts=2026-05-12T00:34:37.749903568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:34:37.752020433Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:34:37.753852937Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:34:37.753873846Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:34:38.487172046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:34:47.156004753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:35:08.488307268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:35:13.69339194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:44388"
ts=2026-05-12T00:35:21.530515809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46872"
ts=2026-05-12T00:35:38.490843201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:35:43.694592351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51462"
level=debug ts=2026-05-12T00:35:44.449225016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:36:08.492960575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:36:13.696410285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:39920"
level=debug ts=2026-05-12T00:36:33.686112203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:36:38.494765385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:37:08.496581313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:37:13.698866485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34922"
ts=2026-05-12T00:37:21.537867604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39390"
ts=2026-05-12T00:37:30.411930996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59688"
level=debug ts=2026-05-12T00:37:35.620411779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:37:38.499239483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:38:08.502197784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:38:30.420501606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39438"
level=debug ts=2026-05-12T00:38:37.41231719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:38:37.414278982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:38:38.50479602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:39:00.423904292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53282"
ts=2026-05-12T00:39:08.506456555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:39:13.703765755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48798"
level=debug ts=2026-05-12T00:39:33.9590568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:39:38.508940674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:39:51.546517566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50448"
ts=2026-05-12T00:40:08.51086846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:40:21.547310136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51520"
level=debug ts=2026-05-12T00:40:32.225894817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:40:38.512783232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:41:06.691167678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:41:08.51530674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:41:38.517945924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:41:51.552685917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51752"
level=debug ts=2026-05-12T00:42:01.923599657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:42:08.520273738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:42:30.448900687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42630"
ts=2026-05-12T00:42:38.522185436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:42:39.79482902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:42:39.796563014Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:42:43.717994585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40422"
ts=2026-05-12T00:43:00.451998506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46098"
ts=2026-05-12T00:43:08.525145591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:43:36.476461041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:43:38.526894274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:43:43.720453734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:56130"
ts=2026-05-12T00:43:51.557594165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53664"
ts=2026-05-12T00:44:08.529346559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:44:30.460698067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51608"
level=debug ts=2026-05-12T00:44:30.733602722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:44:37.748217356Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:44:37.750004039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:44:37.751687403Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:44:37.751708827Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:44:38.531554067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:45:08.532956451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:45:21.562053964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56596"
level=debug ts=2026-05-12T00:45:32.195327707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:45:38.535619292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:46:08.538034958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:46:21.567428312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34148"
level=debug ts=2026-05-12T00:46:33.617251051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:46:38.539192829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:47:08.541534296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:47:09.185059325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:47:09.186872658Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:47:38.544680291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:47:46.341778761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:48:08.546819327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:48:32.870954882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:48:38.549347136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:48:51.576567796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54960"
ts=2026-05-12T00:49:08.550384825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:49:13.746426496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45812"
level=debug ts=2026-05-12T00:49:17.189119978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:49:30.500833588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50076"
ts=2026-05-12T00:49:38.552773719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:49:56.194096729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:50:08.555904417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:50:34.960087133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:50:34.961731432Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:50:38.558935917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:51:08.560921341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:51:30.513370439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49202"
level=debug ts=2026-05-12T00:51:36.068378089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:51:38.561905388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:52:08.563596459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:52:18.359750801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:52:21.593088471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42116"
ts=2026-05-12T00:52:30.521951042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44574"
ts=2026-05-12T00:52:38.565190002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:53:00.526088869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51292"
ts=2026-05-12T00:53:08.565994647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:53:10.508729854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:53:13.766627596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53090"
ts=2026-05-12T00:53:21.59821365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40566"
ts=2026-05-12T00:53:38.567719614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:53:43.767802717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33980"
ts=2026-05-12T00:54:08.569169166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:54:09.318827288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:54:30.537930016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37712"
level=error ts=2026-05-12T00:54:37.751051319Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:54:37.752798297Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:54:37.75450664Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T00:54:37.754528509Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:54:38.570474703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T00:54:43.770641486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38146"
level=debug ts=2026-05-12T00:54:51.34613917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:54:51.347853877Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:54:51.605513945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46318"
ts=2026-05-12T00:55:00.540774694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:32926"
ts=2026-05-12T00:55:08.572236075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:55:13.772118081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49228"
ts=2026-05-12T00:55:21.606485043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33752"
ts=2026-05-12T00:55:38.573174646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:55:39.914990126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:55:51.607866926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60070"
ts=2026-05-12T00:56:08.575166736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T00:56:28.312491417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:56:38.57779791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:56:43.777112316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43712"
ts=2026-05-12T00:57:08.580827306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:57:16.1446515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:57:21.614280985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47260"
ts=2026-05-12T00:57:38.58204666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T00:57:59.755382073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T00:58:08.583120655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T00:58:13.781742117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33006"
ts=2026-05-12T00:58:38.584331782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:58:52.266413355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T00:58:52.268121137Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T00:59:08.586882457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T00:59:38.589427719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T00:59:38.856861096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:00:08.590930011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T01:00:12.903508852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:00:13.790032821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45196"
ts=2026-05-12T01:00:30.587168242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34418"
ts=2026-05-12T01:00:38.593245404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:01:08.595120998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T01:01:14.884532297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:01:21.633254084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49430"
ts=2026-05-12T01:01:38.596253122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T01:02:03.781218281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:02:08.597262083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T01:02:13.798190094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59840"
ts=2026-05-12T01:02:30.602526892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51778"
ts=2026-05-12T01:02:38.598610079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T01:03:00.419976723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T01:03:00.421492739Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T01:03:08.601319536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:03:30.612034551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49028"
level=debug ts=2026-05-12T01:03:36.804904794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:03:38.603846866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T01:03:51.646828274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41366"
ts=2026-05-12T01:04:08.605534859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:04:13.809326466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52908"
level=debug ts=2026-05-12T01:04:18.142942923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T01:04:37.758194804Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T01:04:37.760087462Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T01:04:37.761793324Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T01:04:37.761817197Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T01:04:38.60708821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T01:05:08.608774391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T01:05:16.326590029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:05:21.651581918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36454"
ts=2026-05-12T01:05:30.626875229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51896"
ts=2026-05-12T01:05:38.611434629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
level=debug ts=2026-05-12T01:05:50.579098104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:05:51.653088991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33750"
ts=2026-05-12T01:06:08.61414153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:06:13.816351597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49014"
ts=2026-05-12T01:06:21.654049643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59904"
level=debug ts=2026-05-12T01:06:33.537407772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T01:06:33.539304812Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T01:06:38.617050247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:06:43.818074909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46666"
ts=2026-05-12T01:07:08.619754117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
level=debug ts=2026-05-12T01:07:29.995087653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:07:38.620865395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T01:07:43.820902138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34162"
ts=2026-05-12T01:07:51.662977674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59688"
ts=2026-05-12T01:08:00.644375862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33342"
ts=2026-05-12T01:08:08.622422744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
level=debug ts=2026-05-12T01:08:08.728884134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:08:38.624732371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:09:00.653422666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39936"
level=debug ts=2026-05-12T01:09:08.241052698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:09:08.62906439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-797ea65e 10.129.0.76:7946"
ts=2026-05-12T01:09:21.670610439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52836"
ts=2026-05-12T01:09:38.635704564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T01:09:43.826708865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35650"
ts=2026-05-12T01:09:51.672034475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39814"
level=debug ts=2026-05-12T01:10:03.496853532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:10:08.637010771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T01:10:30.664350547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40336"
ts=2026-05-12T01:10:38.638685265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-9nw4d-a4df8a96 10.130.0.80:7946"
ts=2026-05-12T01:10:43.829074373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58084"
level=debug ts=2026-05-12T01:10:49.561154486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T01:10:49.562722091Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T01:11:08.640187545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T01:11:13.829995529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42184"
level=debug ts=2026-05-12T01:11:35.432517629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T01:11:38.641846255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"
ts=2026-05-12T01:11:43.831135044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53598"
ts=2026-05-12T01:12:08.642883412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-hsm88-e4e43966 10.130.0.79:7946"