level=info ts=2026-04-30T22:50:25.78646742Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-30T22:50:25.786495156Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-30T22:50:25.789989272Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-30T22:50:25.790339836Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T22:50:25.790413098Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=debug ts=2026-04-30T22:50:25.790777503Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T22:50:25.790834035Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.77 inf=eth0
level=info ts=2026-04-30T22:50:25.790867362Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-ef05775a
ts=2026-04-30T22:50:25.790926088Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-30T22:50:25.790936362Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=debug ts=2026-04-30T22:50:25.790954796Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.77 advertisePort=7946
level=info ts=2026-04-30T22:50:25.791121205Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-30T22:50:25.795504978Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T22:50:25.795523458Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-30T22:50:25.795544003Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.424265ms
level=error ts=2026-04-30T22:50:25.79555484Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-30T22:50:25.795568612Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-30T22:50:25.798014988Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T22:50:25.798027608Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-30T22:50:25.804934748Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-30T22:50:25.804987584Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-30T22:50:25.805017646Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-30T22:50:25.805032635Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-30T22:50:25.805061319Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-30T22:50:25.805105922Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-30T22:50:25.805117631Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-30T22:50:25.805302704Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-30T22:50:25.805322892Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-30T22:50:25.805328797Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-30T22:50:25.805388624Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-30T22:50:25.805409031Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-30T22:50:25.805410761Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-30T22:50:25.805532321Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-30T22:50:25.805541133Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-30T22:50:25.805554977Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-30T22:50:26.805917616Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-30T22:50:26.933219673Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-30T22:50:26.93328349Z caller=loki.go:615 msg="Loki started" startup_time=1.223850651s
ts=2026-04-30T22:50:27.601711175Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T22:50:27.601735308Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:50:29.761379997Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T22:50:29.761403743Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-30T22:50:31.933582774Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-30T22:50:31.933638886Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-30T22:50:35.395692931Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T22:50:35.395715914Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-30T22:50:44.876153468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:50:47.167430014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:49464"
ts=2026-04-30T22:50:47.627161619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38056"
ts=2026-04-30T22:50:47.679020024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45690"
ts=2026-04-30T22:50:49.965714409Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.83:7946: dial tcp 10.128.0.83:7946: connect: connection refused"
ts=2026-04-30T22:50:49.966633828Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.81:7946: dial tcp 10.130.0.81:7946: connect: connection refused"
ts=2026-04-30T22:50:49.96751843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-04-30T22:50:49.969630074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-30T22:50:49.970837778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
ts=2026-04-30T22:50:49.970894744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39510"
ts=2026-04-30T22:50:49.971956433Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.72:7946: dial tcp 10.129.0.72:7946: connect: connection refused"
level=info ts=2026-04-30T22:50:49.971967339Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.176391961s
ts=2026-04-30T22:50:57.20713306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T22:51:00.215271704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:51:07.650153237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39820"
level=debug ts=2026-04-30T22:51:14.442450641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:51:27.208965061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T22:51:36.340138681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:51:57.211538046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T22:52:06.021107707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T22:52:06.022636643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T22:52:07.027076078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:60238"
ts=2026-04-30T22:52:18.748293787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:56792"
ts=2026-04-30T22:52:27.214378913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T22:52:37.030036671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:39648"
level=debug ts=2026-04-30T22:52:47.237191417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:52:57.216270265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T22:53:07.661134735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39992"
ts=2026-04-30T22:53:18.753735574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36424"
ts=2026-04-30T22:53:27.217948698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T22:53:37.035759901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:32898"
ts=2026-04-30T22:53:57.221118116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T22:54:01.035666709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:54:07.038865088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:46292"
ts=2026-04-30T22:54:07.66575416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36994"
ts=2026-04-30T22:54:27.22344704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T22:54:48.759435533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36482"
ts=2026-04-30T22:54:57.226250752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T22:54:58.481439714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:55:07.044691727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:51660"
ts=2026-04-30T22:55:07.67032195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57924"
ts=2026-04-30T22:55:27.228241816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T22:55:57.230101226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T22:56:18.027869855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T22:56:18.765426827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51464"
ts=2026-04-30T22:56:27.233058683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T22:56:57.236198457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T22:57:18.770858017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42804"
level=debug ts=2026-04-30T22:57:27.063699244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T22:57:27.065365718Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T22:57:27.238913222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T22:57:37.055261492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53392"
ts=2026-04-30T22:57:57.24173451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T22:58:07.685283546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48740"
level=debug ts=2026-04-30T22:58:10.204850516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T22:58:18.77707466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38430"
ts=2026-04-30T22:58:27.244764135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T22:58:46.897584976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T22:58:57.24682295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T22:59:27.248532078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T22:59:45.537850823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T22:59:57.251300789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:00:18.787560516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34714"
ts=2026-04-30T23:00:27.254113371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:00:31.610771149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-30T23:00:31.934505159Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-30T23:00:31.936792442Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:00:31.938614986Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:00:31.940317694Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:00:31.940347874Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T23:00:31.940365325Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-30T23:00:37.696902081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35152"
ts=2026-04-30T23:00:48.79034278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50450"
ts=2026-04-30T23:00:57.257120127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:01:07.071997332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:59762"
level=debug ts=2026-04-30T23:01:21.363481171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:01:21.365405705Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:01:27.259648874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:01:37.074643271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36930"
ts=2026-04-30T23:01:37.700357268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46400"
ts=2026-04-30T23:01:57.261262857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:01:57.765735667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:02:27.263278822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:02:37.07948843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:41036"
ts=2026-04-30T23:02:37.70430556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51342"
level=debug ts=2026-04-30T23:02:43.013460577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:02:57.265702261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:03:07.08207792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43220"
ts=2026-04-30T23:03:18.803625405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34740"
ts=2026-04-30T23:03:27.267488014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:03:42.914959828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:03:48.806714498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:52964"
ts=2026-04-30T23:03:57.268947157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:04:07.711323457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54442"
ts=2026-04-30T23:04:27.271097915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:04:32.786822382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:04:48.811246541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38214"
ts=2026-04-30T23:04:57.273894647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:05:09.743510844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:05:09.74550449Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:05:27.276997002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:05:57.280440631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:06:02.719106333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:06:27.28367984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:06:37.097448168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:54844"
level=debug ts=2026-04-30T23:06:53.835967744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:06:57.28724408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:07:18.825168786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:50484"
ts=2026-04-30T23:07:27.290152666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:07:37.734621177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58482"
level=debug ts=2026-04-30T23:07:45.032889071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:07:57.29265455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:08:07.106223386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:54498"
ts=2026-04-30T23:08:18.832390489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54994"
ts=2026-04-30T23:08:27.295568477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:08:43.279352998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:08:57.298794606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:09:21.717830281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:09:21.720103917Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:09:27.301901067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:09:37.121986082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53248"
ts=2026-04-30T23:09:48.83952793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:53802"
ts=2026-04-30T23:09:57.305166604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:10:00.473948706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:10:07.125100288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:44136"
ts=2026-04-30T23:10:07.748958053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60368"
ts=2026-04-30T23:10:18.842141717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51634"
ts=2026-04-30T23:10:27.308341689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=error ts=2026-04-30T23:10:31.945063062Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:10:31.947014176Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:10:31.948818261Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:10:31.948840999Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:10:57.311120601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:10:59.668113602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:11:07.129619385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:45086"
ts=2026-04-30T23:11:18.847670082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48684"
ts=2026-04-30T23:11:27.314482403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:11:45.251735702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:11:48.85090829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60904"
ts=2026-04-30T23:11:57.317643014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:12:07.13367196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:48928"
level=debug ts=2026-04-30T23:12:20.983644626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:12:27.320922971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:12:57.323668744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:13:07.766408324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50054"
level=debug ts=2026-04-30T23:13:15.264983096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:13:15.266528011Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:13:27.326608989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:13:57.329615213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:14:01.471908135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:14:18.861616687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38090"
ts=2026-04-30T23:14:27.331451516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:14:37.144689441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:47696"
ts=2026-04-30T23:14:37.773691571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58980"
level=debug ts=2026-04-30T23:14:39.207270571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:14:57.332482704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:15:16.264088161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:15:27.333621874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:15:37.777957642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52710"
ts=2026-04-30T23:15:48.86822756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48482"
level=debug ts=2026-04-30T23:15:53.073275779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:15:57.33598354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:16:27.338140569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:16:29.67067818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:16:29.672459304Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:16:48.874065023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41210"
ts=2026-04-30T23:16:57.34042742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:17:17.7721596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:17:27.341972099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:17:37.16125225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:54100"
level=debug ts=2026-04-30T23:17:54.56546916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:17:57.343341983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:18:07.791280617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45960"
ts=2026-04-30T23:18:18.882458394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43244"
ts=2026-04-30T23:18:27.345815624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:18:29.987370182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:18:48.885322352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46204"
ts=2026-04-30T23:18:57.349983735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:19:18.339874973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:19:18.889824948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:33724"
ts=2026-04-30T23:19:27.352577438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:19:57.354142118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:19:58.219439313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:19:58.221530269Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:20:27.355718993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=error ts=2026-04-30T23:20:31.944888267Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:20:31.946678302Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:20:31.948488424Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:20:31.948508712Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:20:37.80817214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38834"
level=debug ts=2026-04-30T23:20:51.458117793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:20:57.35827859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:21:07.180942949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38410"
ts=2026-04-30T23:21:27.360518026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:21:45.166247891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:21:48.90386313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45170"
ts=2026-04-30T23:21:57.361488274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:22:07.188835272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:34784"
ts=2026-04-30T23:22:18.915190437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40224"
ts=2026-04-30T23:22:27.363299158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:22:37.191911389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:55770"
level=debug ts=2026-04-30T23:22:41.835149463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:22:48.918123453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34628"
ts=2026-04-30T23:22:57.365608409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:23:07.837092911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58072"
level=debug ts=2026-04-30T23:23:20.693068634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:23:27.36851089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:23:37.197664761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43254"
ts=2026-04-30T23:23:57.370716122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:24:07.20087467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58334"
level=debug ts=2026-04-30T23:24:07.277865306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:24:07.279450673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:24:07.841902653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33268"
ts=2026-04-30T23:24:27.373317575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:24:57.378999175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:25:06.317528223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:25:18.93198821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37378"
ts=2026-04-30T23:25:27.38216427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:25:48.934762074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59270"
ts=2026-04-30T23:25:57.384216525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:26:02.470323773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:26:07.851310667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47072"
ts=2026-04-30T23:26:27.385674625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:26:37.213096285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:45218"
level=debug ts=2026-04-30T23:26:43.516627373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:26:48.939105902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55958"
ts=2026-04-30T23:26:57.386948171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:27:07.855949241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46662"
ts=2026-04-30T23:27:27.389308137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:27:37.217350115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:42778"
level=debug ts=2026-04-30T23:27:38.313534676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:27:57.391309055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:28:24.50243934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:28:24.503910257Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:28:27.393778047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:28:37.222716452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:51480"
ts=2026-04-30T23:28:57.396664708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:29:06.448011971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:29:07.866468984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56556"
ts=2026-04-30T23:29:18.949859674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40394"
ts=2026-04-30T23:29:27.399680311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:29:41.475173452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:29:57.401837307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:30:25.383312305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:30:27.404032282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=error ts=2026-04-30T23:30:31.943082155Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:30:31.944681273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:30:31.946189253Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:30:31.946210691Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:30:37.873378059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42104"
ts=2026-04-30T23:30:57.406783034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:31:08.259813014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:31:18.959824383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40980"
ts=2026-04-30T23:31:27.40915207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:31:37.237172463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35604"
ts=2026-04-30T23:31:48.962281585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40832"
level=debug ts=2026-04-30T23:31:56.905219298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:31:56.906901375Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:31:57.412393155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:32:18.965183207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46668"
ts=2026-04-30T23:32:27.415519987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:32:37.882775182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44156"
level=debug ts=2026-04-30T23:32:41.449734514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:32:57.418892512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:33:07.243704626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:60962"
ts=2026-04-30T23:33:07.883858712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54904"
ts=2026-04-30T23:33:18.970610934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57322"
level=debug ts=2026-04-30T23:33:24.38857049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:33:27.420786411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:33:48.973295841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57032"
ts=2026-04-30T23:33:57.422638732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:33:58.955381269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:34:18.9756124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:46232"
ts=2026-04-30T23:34:27.42524431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:34:37.249567108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38452"
level=debug ts=2026-04-30T23:34:52.58003011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:34:57.42810634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:35:07.892871895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41802"
ts=2026-04-30T23:35:27.430336575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:35:47.681362027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:35:47.682830363Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:35:57.432762375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:36:07.897439365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35760"
level=debug ts=2026-04-30T23:36:23.891313751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:36:27.434858649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:36:48.983982065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34610"
ts=2026-04-30T23:36:57.43721121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:37:04.240288342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:37:07.901596789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47336"
ts=2026-04-30T23:37:18.986277484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37520"
ts=2026-04-30T23:37:27.439760188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:37:57.442489479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:38:01.48282008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:38:18.99138189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35918"
ts=2026-04-30T23:38:27.444286046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:38:37.908641186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39090"
ts=2026-04-30T23:38:48.994358149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40120"
level=debug ts=2026-04-30T23:38:55.61850648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:38:57.446134207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:39:07.909618859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44284"
ts=2026-04-30T23:39:27.447162587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:39:37.275515517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:46108"
ts=2026-04-30T23:39:37.911137665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35752"
level=debug ts=2026-04-30T23:39:38.219160753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:39:38.220872514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:39:57.449196206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:40:27.450968512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=error ts=2026-04-30T23:40:31.942801716Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:40:31.944402783Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:40:31.945924013Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:40:31.94594664Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-30T23:40:32.238797499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:40:57.453522977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:41:07.284198804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:39618"
level=debug ts=2026-04-30T23:41:17.719212675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:41:19.0022863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54852"
ts=2026-04-30T23:41:27.456725031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:41:49.004467038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40690"
ts=2026-04-30T23:41:57.459809996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:42:18.225788622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:42:27.463011909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:42:37.291227713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:50578"
ts=2026-04-30T23:42:57.46607488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:43:18.391607529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:43:27.468841714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:43:57.47051801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:44:07.934943979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34688"
level=debug ts=2026-04-30T23:44:17.858543116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:44:17.860337478Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:44:19.014441015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:38522"
ts=2026-04-30T23:44:27.47147967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:44:49.016907022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:55646"
ts=2026-04-30T23:44:57.473756531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:45:07.137332289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:45:07.301822046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36972"
ts=2026-04-30T23:45:27.475541676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:45:49.022196094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:39484"
ts=2026-04-30T23:45:57.477864564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:45:57.517592037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:46:07.305747831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:39122"
ts=2026-04-30T23:46:27.479880212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:46:52.76265298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:46:57.482178919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:47:19.030026431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:57832"
ts=2026-04-30T23:47:27.484858004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:47:32.390965752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:47:57.487377432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:48:07.313408299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:47104"
ts=2026-04-30T23:48:27.490180398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-04-30T23:48:29.458623845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:48:29.460245574Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:48:49.038329393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59598"
ts=2026-04-30T23:48:57.492504855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:49:07.317003679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35358"
level=debug ts=2026-04-30T23:49:11.303746281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:49:27.494106662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:49:49.042198973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:56582"
ts=2026-04-30T23:49:57.496488482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:50:04.245184665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:50:19.044954712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51854"
ts=2026-04-30T23:50:27.499717184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=error ts=2026-04-30T23:50:31.94400344Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:50:31.945654312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:50:31.947155552Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T23:50:31.947179315Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:50:37.325097495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:57422"
level=debug ts=2026-04-30T23:50:44.414163537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:50:57.501914969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:51:07.973983331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33758"
level=debug ts=2026-04-30T23:51:19.907916282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:51:27.50365193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-04-30T23:51:57.505183103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:52:01.497040918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:52:01.498563515Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:52:19.052172236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:42772"
ts=2026-04-30T23:52:27.507812595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:52:37.981812579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49186"
level=debug ts=2026-04-30T23:52:52.234529025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:52:57.510535503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:53:27.513300412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:53:38.184038117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:53:57.5162162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:54:27.518903044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:54:28.046190932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:54:37.991448919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60348"
ts=2026-04-30T23:54:57.521247991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:55:07.350041008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:46386"
level=debug ts=2026-04-30T23:55:17.639813029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:55:27.523628604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:55:49.067592824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49632"
ts=2026-04-30T23:55:57.527040726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-04-30T23:56:07.998060798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53416"
level=debug ts=2026-04-30T23:56:12.986171635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T23:56:12.987829645Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T23:56:27.529891481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:56:56.697871355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:56:57.532884236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:57:19.075095199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:34668"
ts=2026-04-30T23:57:27.536095947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:57:31.438563923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:57:37.36396341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:55364"
ts=2026-04-30T23:57:57.539127052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:58:07.366746546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:43262"
ts=2026-04-30T23:58:27.54145039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-04-30T23:58:30.520357887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:58:38.010207434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42996"
ts=2026-04-30T23:58:49.08117116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36964"
ts=2026-04-30T23:58:57.543479242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-04-30T23:59:27.546568287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-04-30T23:59:31.718541236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T23:59:38.014157052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52550"
ts=2026-04-30T23:59:49.085388307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45044"
ts=2026-04-30T23:59:57.549148149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:00:08.444430652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:00:08.44611295Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:00:27.551158952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=error ts=2026-05-01T00:00:31.944690909Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:00:31.946931664Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:00:31.949016379Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:00:31.949040276Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:00:37.376485952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:59132"
ts=2026-05-01T00:00:38.017784613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57064"
level=debug ts=2026-05-01T00:00:48.824661474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:00:57.552868183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:01:08.018820064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51318"
ts=2026-05-01T00:01:27.555159246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-05-01T00:01:34.320517322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:01:38.020017231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37316"
ts=2026-05-01T00:01:49.095539484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:44540"
ts=2026-05-01T00:01:57.557221582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:02:08.021359968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58880"
level=debug ts=2026-05-01T00:02:20.347749183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:02:27.559476118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:02:57.561847686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:03:05.918516776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:03:08.025931446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52474"
ts=2026-05-01T00:03:19.1023555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41376"
ts=2026-05-01T00:03:27.564897513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:03:45.173105236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:03:45.174625464Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:03:57.567901635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:04:20.305630792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:04:27.56986317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:04:38.033449503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60574"
ts=2026-05-01T00:04:57.571126322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-05-01T00:05:06.114968047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:05:07.39701785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:34706"
ts=2026-05-01T00:05:08.034524763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40966"
ts=2026-05-01T00:05:19.111757551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41026"
ts=2026-05-01T00:05:27.573163101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:05:57.576483303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:06:03.270208033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:06:07.401659115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35844"
ts=2026-05-01T00:06:19.116005979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58202"
ts=2026-05-01T00:06:27.578710798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:06:38.041893842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45364"
ts=2026-05-01T00:06:49.118474389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48900"
ts=2026-05-01T00:06:57.580898504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:07:01.130090136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:07:08.042794437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37302"
ts=2026-05-01T00:07:19.121139898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:40412"
ts=2026-05-01T00:07:27.58384583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:07:53.847023954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:07:53.848539604Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:07:57.586869969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:08:07.411651312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:33074"
ts=2026-05-01T00:08:08.046795595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50590"
ts=2026-05-01T00:08:27.589327786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:08:37.414484585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:60384"
level=debug ts=2026-05-01T00:08:53.950917891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:08:57.590905178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:09:07.416868268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:35274"
ts=2026-05-01T00:09:27.593518458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:09:39.705975367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:09:49.133226216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37418"
ts=2026-05-01T00:09:57.596163223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:10:07.420800619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:55712"
level=debug ts=2026-05-01T00:10:17.654029652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:10:27.598609893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=error ts=2026-05-01T00:10:31.944690046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:10:31.946504789Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:10:31.94831026Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:10:31.948330982Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:10:49.13762291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:54098"
ts=2026-05-01T00:10:57.601171374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:11:02.47401742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:11:07.425924639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:58704"
ts=2026-05-01T00:11:19.140137784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37158"
ts=2026-05-01T00:11:27.60350289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:11:37.428700601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:56624"
ts=2026-05-01T00:11:49.142289216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:60620"
level=debug ts=2026-05-01T00:11:53.824676491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:11:53.826223881Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:11:57.604715985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:12:07.431408089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:33594"
ts=2026-05-01T00:12:08.067674011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37142"
ts=2026-05-01T00:12:27.606954178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:12:33.649879974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:12:49.148633387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:41780"
ts=2026-05-01T00:12:57.609812099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:13:19.151068059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:45956"
level=debug ts=2026-05-01T00:13:19.852042638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:13:27.612062253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:13:49.153615951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:48748"
level=debug ts=2026-05-01T00:13:56.317660862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:13:57.613423448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:14:07.441645507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:34052"
ts=2026-05-01T00:14:08.077655366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47620"
ts=2026-05-01T00:14:27.615273998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:14:38.078471847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60540"
level=debug ts=2026-05-01T00:14:41.874215173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:14:49.157858256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:35468"
ts=2026-05-01T00:14:57.617733264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-05-01T00:15:21.428586601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:15:21.430307221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:15:27.620123544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:15:57.623033458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:16:07.45148911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:34298"
ts=2026-05-01T00:16:08.085714278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41198"
level=debug ts=2026-05-01T00:16:17.434124855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:16:27.625851938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:16:38.086954885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42968"
ts=2026-05-01T00:16:57.629155357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:17:08.08853832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35024"
level=debug ts=2026-05-01T00:17:13.36496316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:17:19.169607216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:36478"
ts=2026-05-01T00:17:27.632256701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:17:57.634800902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:18:05.05361429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:18:07.458757077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53624"
ts=2026-05-01T00:18:08.093448592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49812"
ts=2026-05-01T00:18:27.637647778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:18:37.461631023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36298"
ts=2026-05-01T00:18:57.640704797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:19:02.038667434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:19:07.464226274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:50036"
ts=2026-05-01T00:19:19.177913734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37972"
ts=2026-05-01T00:19:27.643567577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:19:37.46669972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:57904"
level=debug ts=2026-05-01T00:19:39.909818947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:19:39.911361055Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:19:49.180297371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59244"
ts=2026-05-01T00:19:57.646779152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:20:07.469267543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38560"
ts=2026-05-01T00:20:08.103260141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42728"
ts=2026-05-01T00:20:27.648418707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=error ts=2026-05-01T00:20:31.944683028Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:20:31.946525297Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:20:31.948093373Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T00:20:31.94811942Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T00:20:40.386530553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:20:57.650074556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:21:07.475990197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:40300"
ts=2026-05-01T00:21:19.187858204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:51590"
ts=2026-05-01T00:21:27.652859839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
level=debug ts=2026-05-01T00:21:39.996119349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:21:49.191005071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:39456"
ts=2026-05-01T00:21:57.654735613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:22:08.113129898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49694"
level=debug ts=2026-05-01T00:22:26.419269137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:22:27.656745793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:22:57.658316158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:23:08.116999406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52994"
level=debug ts=2026-05-01T00:23:25.573684256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:23:27.659273649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:23:57.661330204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:24:07.486376726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36262"
level=debug ts=2026-05-01T00:24:18.024331709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T00:24:18.025924093Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T00:24:27.66315933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:24:49.201712301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:47898"
level=debug ts=2026-05-01T00:24:55.604834716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:24:57.664329537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
ts=2026-05-01T00:25:27.666148916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
level=debug ts=2026-05-01T00:25:31.49907937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:25:37.494504891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:49678"
ts=2026-05-01T00:25:57.668906758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:26:07.496754547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:53038"
ts=2026-05-01T00:26:27.67127235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-74d70049 10.129.0.76:7946"
level=debug ts=2026-05-01T00:26:31.993447341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:26:38.135686856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33586"
ts=2026-05-01T00:26:57.672892904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-6j2qm-1cd8992e 10.130.0.82:7946"
ts=2026-05-01T00:27:07.500795101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:54668"
level=debug ts=2026-05-01T00:27:11.904088515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T00:27:19.213725247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:43142"
ts=2026-05-01T00:27:27.675829475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cf9w7-7c0ae822 10.130.0.80:7946"
ts=2026-05-01T00:27:38.140220592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43896"