level=info ts=2026-04-30T14:35:08.766877598Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-30T14:35:08.766932808Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-30T14:35:08.768428097Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-30T14:35:08.768852603Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T14:35:08.768963472Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.77 inf=eth0
level=debug ts=2026-04-30T14:35:08.770336903Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-30T14:35:08.770416866Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.77 inf=eth0
level=info ts=2026-04-30T14:35:08.770429902Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-860a8afe
ts=2026-04-30T14:35:08.770485753Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-30T14:35:08.770495654Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.77 advertisePort=7946
level=debug ts=2026-04-30T14:35:08.770520906Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.77 advertisePort=7946
level=info ts=2026-04-30T14:35:08.770632095Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-30T14:35:08.775127546Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T14:35:08.775140058Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-30T14:35:08.775149349Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.519045ms
level=error ts=2026-04-30T14:35:08.775158342Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-30T14:35:08.775170087Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-30T14:35:08.77706143Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T14:35:08.777075775Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-30T14:35:08.78414781Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-30T14:35:08.784146094Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-30T14:35:08.784171597Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-30T14:35:08.78417858Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-30T14:35:08.784177889Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-30T14:35:08.78419592Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-30T14:35:08.784389601Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-30T14:35:08.784399161Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-30T14:35:08.784402492Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-30T14:35:08.784405206Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-30T14:35:08.784718019Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-30T14:35:08.784724194Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-30T14:35:08.784739629Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-30T14:35:08.784872091Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-30T14:35:08.784882605Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-30T14:35:08.784892409Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-30T14:35:09.785662611Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-30T14:35:09.922927949Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-30T14:35:09.922978332Z caller=loki.go:615 msg="Loki started" startup_time=1.222819154s
ts=2026-04-30T14:35:10.071936866Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T14:35:10.071965008Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:35:12.526733606Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T14:35:12.526762787Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-30T14:35:14.923871199Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-30T14:35:14.923941399Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-30T14:35:19.4769962Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-30T14:35:19.477025422Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-30T14:35:27.39360125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:35:28.370973211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48884"
ts=2026-04-30T14:35:31.587625573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.70:7946"
ts=2026-04-30T14:35:31.588971095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-30T14:35:31.590281296Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-04-30T14:35:31.590887778Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.74:7946: dial tcp 10.130.0.74:7946: connect: connection refused"
ts=2026-04-30T14:35:31.590972621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.77:7946"
ts=2026-04-30T14:35:31.59102771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36058"
level=info ts=2026-04-30T14:35:31.591250143Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=22.816073626s
ts=2026-04-30T14:35:33.351471408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:38898"
ts=2026-04-30T14:35:34.947617025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46046"
level=debug ts=2026-04-30T14:35:44.591818015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:35:49.810047833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47564"
ts=2026-04-30T14:35:52.510794346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:36:00.557227614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44252"
level=debug ts=2026-04-30T14:36:08.569993081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:36:13.955927311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:58854"
ts=2026-04-30T14:36:19.811922551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45744"
ts=2026-04-30T14:36:22.512800592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:36:29.124720658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:36:52.516297633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:37:02.340709238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T14:37:02.342527202Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:37:13.958970257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:59630"
ts=2026-04-30T14:37:19.817266236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41872"
ts=2026-04-30T14:37:22.519932954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:37:44.083798856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:37:49.818432178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33812"
ts=2026-04-30T14:37:52.522248044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:38:00.571792557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36282"
ts=2026-04-30T14:38:13.961867986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:45508"
ts=2026-04-30T14:38:22.523342518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:38:46.78036403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:38:49.820680643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59140"
ts=2026-04-30T14:38:52.524626536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:39:13.966215518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:32830"
ts=2026-04-30T14:39:22.526216475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:39:49.824210505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:32828"
ts=2026-04-30T14:39:52.526997546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:40:00.784979328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:40:22.528227355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:40:43.972066936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:35886"
ts=2026-04-30T14:40:52.529529601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:40:58.666831657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-30T14:41:19.832138775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53426"
ts=2026-04-30T14:41:22.531198217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:41:49.833359983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44690"
ts=2026-04-30T14:41:52.532385778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:42:14.207879477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-30T14:42:14.209362551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:42:22.534804684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:42:30.602542882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55654"
ts=2026-04-30T14:42:52.536818668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:42:55.49015614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:43:00.605479731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59722"
ts=2026-04-30T14:43:13.983094454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43632"
ts=2026-04-30T14:43:22.537953165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:43:43.985115719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:48362"
ts=2026-04-30T14:43:52.539201614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:43:56.078087614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:44:22.540321446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:44:30.61431771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47826"
level=debug ts=2026-04-30T14:44:37.395232434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:44:52.54316058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=info ts=2026-04-30T14:45:14.924362439Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-30T14:45:14.928145689Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:45:14.930213238Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:45:14.932165531Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:45:14.932194989Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-30T14:45:14.932207464Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-30T14:45:22.545306926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:45:33.547304407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:45:49.858507604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60006"
ts=2026-04-30T14:45:52.54653126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:46:19.860422632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45446"
ts=2026-04-30T14:46:22.549463953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:46:29.460123053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T14:46:29.46232066Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:46:52.552258401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:47:22.553798258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:47:23.374737257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:47:30.634113039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48276"
ts=2026-04-30T14:47:49.866885344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34752"
ts=2026-04-30T14:47:52.556275506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:48:00.637071682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34260"
level=debug ts=2026-04-30T14:48:07.435980871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:48:14.012990409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:59144"
ts=2026-04-30T14:48:19.868472409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60706"
ts=2026-04-30T14:48:22.558260221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:48:52.559893658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:49:02.673138335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:49:22.561281601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:49:30.647418265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34588"
level=debug ts=2026-04-30T14:49:51.099092194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:49:52.562297106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T14:50:22.563281054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:50:43.584681853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T14:50:43.586633537Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:50:44.023754082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40800"
ts=2026-04-30T14:50:52.565048865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:51:19.755081725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:51:19.881574033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36768"
ts=2026-04-30T14:51:22.56751126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:51:52.569205798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:52:14.031953107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:34892"
level=debug ts=2026-04-30T14:52:18.877269411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:52:22.57045208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:52:30.667377371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57840"
ts=2026-04-30T14:52:52.571987774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:53:06.06104219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:53:14.035421633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:40552"
ts=2026-04-30T14:53:19.890475302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41094"
ts=2026-04-30T14:53:22.573259676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:53:52.575591714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:53:57.360259802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:54:19.895179581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52632"
ts=2026-04-30T14:54:22.577648955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:54:37.716572535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T14:54:37.718629858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:54:44.040978606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:36980"
ts=2026-04-30T14:54:52.578972365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=error ts=2026-04-30T14:55:14.936410705Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:55:14.938481348Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:55:14.940435419Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T14:55:14.940459022Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:55:22.580353058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T14:55:23.54766767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:55:44.044897739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:46738"
ts=2026-04-30T14:55:52.582360367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:56:00.689468238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56634"
level=debug ts=2026-04-30T14:56:15.949785994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:56:22.584083252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:56:44.049399308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:59366"
ts=2026-04-30T14:56:49.906143547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42910"
ts=2026-04-30T14:56:52.585384128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:57:09.741147553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:57:22.588016461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:57:49.909596892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38976"
ts=2026-04-30T14:57:52.591616897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:58:07.507377577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T14:58:22.594082308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T14:58:49.912703829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60420"
ts=2026-04-30T14:58:52.596824408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T14:58:59.946325921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T14:58:59.948050625Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T14:59:22.600588178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T14:59:30.712314911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:35998"
ts=2026-04-30T14:59:44.065096462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:52492"
ts=2026-04-30T14:59:52.60234443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T14:59:53.541520388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:00:00.715188236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48796"
ts=2026-04-30T15:00:22.603511696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:00:36.332468126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:00:44.069486854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:37750"
ts=2026-04-30T15:00:52.606041483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:01:12.155969608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:01:22.608881266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:01:52.611045331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:02:01.491269257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:02:14.075635458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:51516"
ts=2026-04-30T15:02:22.612172846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:02:49.930131451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42238"
ts=2026-04-30T15:02:52.613386886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:03:01.534477556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:03:01.536828638Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:03:19.931579957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45230"
ts=2026-04-30T15:03:22.614588739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:03:30.736271659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41426"
level=debug ts=2026-04-30T15:03:47.266475444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:03:52.617034648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:04:00.741060396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59282"
ts=2026-04-30T15:04:22.619519353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:04:45.18685785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:04:52.621840666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:05:14.089735389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53128"
level=error ts=2026-04-30T15:05:14.936844374Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:05:14.938491871Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:05:14.940108046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:05:14.940131096Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:05:19.939557127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42200"
ts=2026-04-30T15:05:22.623457936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:05:30.751004685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50772"
level=debug ts=2026-04-30T15:05:43.13186888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:05:52.625836422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:06:14.094322407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:35554"
level=debug ts=2026-04-30T15:06:21.157194358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:06:22.628399732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:06:44.096009318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:53366"
ts=2026-04-30T15:06:52.63006476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:07:01.408936309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:07:01.410550195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:07:14.097686372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55612"
ts=2026-04-30T15:07:19.946790172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46930"
ts=2026-04-30T15:07:22.63087538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:07:35.896689959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:07:52.631903732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:08:14.10313954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:43102"
ts=2026-04-30T15:08:22.634148313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:08:35.827100835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:08:52.637599203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:09:22.640374603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:09:30.776301013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44124"
level=debug ts=2026-04-30T15:09:34.524343123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:09:52.64137051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:10:19.957551699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35046"
ts=2026-04-30T15:10:22.643832205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:10:29.888532858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:10:49.959148738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59294"
ts=2026-04-30T15:10:52.645541712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:11:00.786185315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38160"
level=debug ts=2026-04-30T15:11:21.187357053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:11:21.189083538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:11:22.646944943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:11:30.789142243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:60344"
ts=2026-04-30T15:11:44.117053357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:56236"
ts=2026-04-30T15:11:52.647975364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:11:58.295035681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:12:22.649241979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:12:30.795097212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38886"
level=debug ts=2026-04-30T15:12:33.966170753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:12:52.650131763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:13:00.798420681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57298"
level=debug ts=2026-04-30T15:13:15.245227627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:13:22.652631755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:13:52.656270181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:13:56.729405771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:14:19.975561681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35614"
ts=2026-04-30T15:14:22.658080392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:14:30.807361826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55966"
level=debug ts=2026-04-30T15:14:47.593424409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:14:47.595349569Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:14:49.976841341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42104"
ts=2026-04-30T15:14:52.659614229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:15:00.810851934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58084"
level=error ts=2026-04-30T15:15:14.935764865Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:15:14.93752009Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:15:14.939194233Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:15:14.939218737Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:15:19.97767164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51136"
ts=2026-04-30T15:15:22.661422202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:15:47.450951788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:15:52.662764919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:16:19.982169287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53466"
ts=2026-04-30T15:16:22.663508452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:16:42.762033916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:16:52.665552885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:17:20.047747765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:17:22.667366849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:17:30.826487881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34876"
ts=2026-04-30T15:17:44.141117084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:57664"
ts=2026-04-30T15:17:52.669083025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:18:00.798885347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:18:19.989703222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48940"
ts=2026-04-30T15:18:22.671452849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:18:49.990760459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46898"
ts=2026-04-30T15:18:52.673695742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:18:54.719701193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:18:54.721356404Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:19:14.145202625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:50118"
ts=2026-04-30T15:19:22.675094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:19:31.489735574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:19:44.146634623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:47426"
ts=2026-04-30T15:19:52.67664503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:20:00.843138901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41640"
ts=2026-04-30T15:20:19.996421724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33744"
ts=2026-04-30T15:20:22.678241567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:20:26.573420148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:20:44.149146566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:35084"
ts=2026-04-30T15:20:52.67983716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:21:00.849516925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44414"
level=debug ts=2026-04-30T15:21:16.813164347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:21:22.681142141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:21:44.153822368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:48506"
ts=2026-04-30T15:21:52.682648925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:22:03.01729461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:22:22.684677306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:22:30.859838875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40160"
level=debug ts=2026-04-30T15:22:41.418049512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:22:41.419535516Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:22:52.68722731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:23:00.862927637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:54774"
level=debug ts=2026-04-30T15:23:17.540135336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:23:22.689284879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:23:52.692601951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:24:15.093153819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:24:22.696272979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:24:30.872872978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53096"
ts=2026-04-30T15:24:52.69784938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
level=debug ts=2026-04-30T15:25:12.169901946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:25:14.166010803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:55012"
level=error ts=2026-04-30T15:25:14.936790728Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:25:14.938949768Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:25:14.940792077Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-30T15:25:14.940828257Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:25:20.014427978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39144"
ts=2026-04-30T15:25:22.700440924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:25:30.880093243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38332"
level=debug ts=2026-04-30T15:25:51.19913407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:25:52.702505492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:26:20.01756964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40992"
ts=2026-04-30T15:26:22.703349809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:26:44.592638692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:26:44.594397819Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:26:52.704758882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:27:20.022617375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43550"
ts=2026-04-30T15:27:22.706491689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:27:30.893549504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50176"
level=debug ts=2026-04-30T15:27:36.759578428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:27:52.709183233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:28:14.175533214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:38634"
level=debug ts=2026-04-30T15:28:18.608117413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:28:20.027003011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42914"
ts=2026-04-30T15:28:22.711845713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"
ts=2026-04-30T15:28:52.713219144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
level=debug ts=2026-04-30T15:29:06.917694069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:29:22.715272241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
ts=2026-04-30T15:29:44.183004621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.75:60430"
ts=2026-04-30T15:29:52.718435469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-w5kc2-70f436c5 10.128.0.70:7946"
level=debug ts=2026-04-30T15:29:53.242956625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-30T15:30:00.909290983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34498"
ts=2026-04-30T15:30:22.720730879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6cdsl-2352200c 10.130.0.72:7946"
ts=2026-04-30T15:30:30.912513952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41642"
level=debug ts=2026-04-30T15:30:44.471684987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-30T15:30:44.473239931Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-30T15:30:52.722194869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dd8cf5a2 10.130.0.75:7946"