level=info ts=2026-04-29T15:01:15.78825392Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-29T15:01:15.788574646Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-29T15:01:15.791109058Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-29T15:01:15.791538058Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T15:01:15.791645098Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.48 inf=eth0
level=debug ts=2026-04-29T15:01:15.792207156Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T15:01:15.792291689Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.48 inf=eth0
level=info ts=2026-04-29T15:01:15.794374848Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b14089bf
ts=2026-04-29T15:01:15.794446353Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-29T15:01:15.794464148Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.48 advertisePort=7946
level=debug ts=2026-04-29T15:01:15.794482273Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.48 advertisePort=7946
level=info ts=2026-04-29T15:01:15.794639483Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-29T15:01:15.807442545Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T15:01:15.807473074Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T15:01:15.807486297Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=12.848964ms
level=error ts=2026-04-29T15:01:15.807499015Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-29T15:01:15.807511649Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-29T15:01:15.811190457Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-29T15:01:15.811386184Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-29T15:01:15.811411301Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-29T15:01:15.811421754Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-29T15:01:15.811770913Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-29T15:01:15.811839093Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-29T15:01:15.811879768Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-29T15:01:15.811902528Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-29T15:01:15.812085874Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-29T15:01:15.812096356Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-29T15:01:15.812147474Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-29T15:01:15.812186581Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-29T15:01:15.812207929Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-29T15:01:15.812408982Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-29T15:01:15.812421352Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-29T15:01:15.812441412Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-29T15:01:15.81277205Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T15:01:15.812791501Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T15:01:16.813177257Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-29T15:01:16.928448854Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-29T15:01:16.928502094Z caller=loki.go:615 msg="Loki started" startup_time=1.226728307s
ts=2026-04-29T15:01:17.73619947Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T15:01:17.736235257Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:01:19.854015685Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T15:01:19.854040874Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T15:01:21.929123849Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-29T15:01:21.929173579Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-29T15:01:26.568730671Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T15:01:26.568752665Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T15:01:35.158670383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:01:36.968442051Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T15:01:36.96846519Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T15:01:55.908789651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:01:56.657716053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.59:7946"
ts=2026-04-29T15:01:56.659564894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.48:7946"
ts=2026-04-29T15:01:56.659587493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.48:56042"
ts=2026-04-29T15:01:56.660732196Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.45:7946: dial tcp 10.130.0.45:7946: connect: connection refused"
ts=2026-04-29T15:01:56.661466333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.43:7946"
ts=2026-04-29T15:01:56.662397081Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.47:7946: dial tcp 10.130.0.47:7946: connect: connection refused"
ts=2026-04-29T15:01:56.66285866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.44:7946"
ts=2026-04-29T15:01:56.66391972Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.46:7946: dial tcp 10.130.0.46:7946: connect: connection refused"
level=info ts=2026-04-29T15:01:56.663929228Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=40.856412053s
ts=2026-04-29T15:01:57.264587059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35596"
ts=2026-04-29T15:01:57.29455197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:44330"
ts=2026-04-29T15:01:58.21374473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:54924"
ts=2026-04-29T15:02:05.15924753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35630"
ts=2026-04-29T15:02:05.857746043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36460"
ts=2026-04-29T15:02:10.942185148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:54778"
level=debug ts=2026-04-29T15:02:14.826172628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:02:17.876226991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:02:36.865652129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:02:47.878008067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:02:57.271390678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48454"
ts=2026-04-29T15:02:57.299995726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:41962"
level=debug ts=2026-04-29T15:03:03.854948021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T15:03:03.857020835Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:03:17.879367935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:03:40.945821607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:52710"
ts=2026-04-29T15:03:47.880279738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:03:52.372622458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:03:57.30277641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:34142"
ts=2026-04-29T15:04:17.88208684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:04:27.304283932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:49394"
ts=2026-04-29T15:04:47.885240416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:04:55.328746585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:05:17.887115955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:05:40.950997872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:54180"
ts=2026-04-29T15:05:47.889105144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:05:57.290572835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58626"
ts=2026-04-29T15:06:10.952763434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46924"
level=debug ts=2026-04-29T15:06:11.005316899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:06:17.890899785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:06:27.293629865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:52190"
ts=2026-04-29T15:06:47.893599336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:07:17.897164115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:07:19.105567314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T15:07:47.900053683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:07:57.318707337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36812"
ts=2026-04-29T15:08:17.902559577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:08:27.307168509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51878"
level=debug ts=2026-04-29T15:08:36.068647357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T15:08:36.070305863Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:08:47.904037546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:09:10.96239906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:43726"
level=debug ts=2026-04-29T15:09:12.806777953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:09:17.905764799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:09:40.963905703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:55052"
ts=2026-04-29T15:09:47.90772996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:10:10.964926015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:50280"
level=debug ts=2026-04-29T15:10:14.735062907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:10:17.910508493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:10:27.31933542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43110"
ts=2026-04-29T15:10:47.912247076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:10:51.68663355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:10:57.330666017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:50258"
ts=2026-04-29T15:11:17.913560976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=info ts=2026-04-29T15:11:21.929368112Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-29T15:11:21.935557909Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:11:21.937278698Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:11:21.939005873Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:11:21.939029327Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T15:11:21.939041125Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-29T15:11:27.325725555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51712"
ts=2026-04-29T15:11:27.332200366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:41826"
level=debug ts=2026-04-29T15:11:47.397932444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:11:47.916325616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:12:17.919159791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:12:27.331412168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45208"
level=debug ts=2026-04-29T15:12:35.784003474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:12:35.785602422Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:12:47.921576319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:12:57.334842628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39072"
ts=2026-04-29T15:13:17.924292601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:13:25.454791068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:13:47.926635492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:13:57.339367344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:47064"
ts=2026-04-29T15:13:57.341499505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55114"
level=debug ts=2026-04-29T15:14:09.70412312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:14:10.983555257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:42358"
ts=2026-04-29T15:14:17.928066654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:14:27.34026386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:54866"
ts=2026-04-29T15:14:27.344817442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39698"
ts=2026-04-29T15:14:40.984475174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:50988"
ts=2026-04-29T15:14:47.930419858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:14:57.342383344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:40482"
ts=2026-04-29T15:14:57.347975247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33590"
level=debug ts=2026-04-29T15:15:01.674079223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:15:10.985635703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:39836"
ts=2026-04-29T15:15:17.933848374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:15:27.34317244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:53342"
ts=2026-04-29T15:15:47.935814846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:15:51.894649616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:16:17.938044378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:16:32.565765929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:16:32.567453823Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:16:47.941565162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:17:10.994004452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:34348"
ts=2026-04-29T15:17:17.943597417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:17:27.350866989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:48966"
ts=2026-04-29T15:17:27.363699502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53542"
level=debug ts=2026-04-29T15:17:31.528482159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:17:40.995596847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:51604"
ts=2026-04-29T15:17:47.946159968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:17:57.35193866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:34234"
ts=2026-04-29T15:18:10.996255423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:55090"
level=debug ts=2026-04-29T15:18:11.293622131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:18:17.948079428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:18:27.36972826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:52440"
ts=2026-04-29T15:18:40.997113166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:50002"
ts=2026-04-29T15:18:47.949202111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:18:55.587937025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:19:17.950263682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:19:40.999726943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:33344"
ts=2026-04-29T15:19:47.951658425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:19:55.227617002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:20:11.000951722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:50248"
ts=2026-04-29T15:20:17.953647063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:20:41.002559705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:39430"
ts=2026-04-29T15:20:47.95624141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:20:50.541260888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:20:50.542868436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:20:57.386116161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:35174"
ts=2026-04-29T15:21:17.959511474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=error ts=2026-04-29T15:21:21.944949937Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:21:21.946626886Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:21:21.948167351Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:21:21.948187549Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:21:27.365705847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:46854"
level=debug ts=2026-04-29T15:21:33.520039655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:21:41.005271741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:45684"
ts=2026-04-29T15:21:47.962312867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:21:57.366654658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:39244"
ts=2026-04-29T15:22:17.966466089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:22:27.396531211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50512"
level=debug ts=2026-04-29T15:22:33.398186674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:22:41.010167553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:59130"
ts=2026-04-29T15:22:47.96810266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:23:11.011317907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:56010"
ts=2026-04-29T15:23:17.970655081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:23:23.154245462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:23:27.372970869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:41558"
ts=2026-04-29T15:23:41.012818897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:41104"
ts=2026-04-29T15:23:47.974186727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:23:57.374900793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:56388"
level=debug ts=2026-04-29T15:24:09.604153466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:24:17.976644109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:24:44.603908601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:24:44.60622484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:24:47.978321598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:24:57.412524762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42178"
ts=2026-04-29T15:25:11.020858944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46438"
ts=2026-04-29T15:25:17.980118338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:25:22.710150535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:25:27.384455115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36810"
ts=2026-04-29T15:25:47.983245689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:25:57.386169674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:37148"
ts=2026-04-29T15:26:11.024546374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46374"
ts=2026-04-29T15:26:17.986935131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:26:18.896889288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:26:27.421267957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59774"
ts=2026-04-29T15:26:47.989055673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:26:57.424735949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34770"
level=debug ts=2026-04-29T15:27:06.400956467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:27:11.0286024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:60526"
ts=2026-04-29T15:27:18.003178118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:27:27.428478754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60098"
ts=2026-04-29T15:27:48.006052682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:27:57.431784896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33582"
level=debug ts=2026-04-29T15:28:02.326387153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:28:18.008401659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:28:48.009510742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:28:51.083666054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:28:51.085560196Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:29:18.01074418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:29:27.441713201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44178"
level=debug ts=2026-04-29T15:29:44.789756827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:29:48.012900876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:30:11.039617573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:51746"
ts=2026-04-29T15:30:18.015328104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:30:43.466199087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:30:48.017061138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:30:57.417154635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:47592"
ts=2026-04-29T15:30:57.451104256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40082"
ts=2026-04-29T15:31:18.018201369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=error ts=2026-04-29T15:31:21.942928047Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:31:21.944754552Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:31:21.946907926Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:31:21.946929568Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T15:31:35.754733475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:31:48.020938841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:31:57.418963665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:46612"
level=debug ts=2026-04-29T15:32:17.396071627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:32:18.022997061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:32:41.049415685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:41350"
ts=2026-04-29T15:32:48.02445755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:32:56.28354965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:32:56.285285752Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:33:11.050571833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:40790"
ts=2026-04-29T15:33:18.026701319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:33:42.092935902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:33:48.027694825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:34:18.028881959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:34:19.149450818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:34:27.42807238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:34224"
ts=2026-04-29T15:34:27.491097105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33984"
ts=2026-04-29T15:34:48.029938891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:35:11.059654883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:53474"
ts=2026-04-29T15:35:18.032538939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:35:20.760089968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:35:41.060572728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:50628"
ts=2026-04-29T15:35:48.036068364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:35:56.876000245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:36:11.061493859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:36596"
ts=2026-04-29T15:36:18.038893882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:36:27.435937894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:52552"
ts=2026-04-29T15:36:48.041943778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:36:51.463567477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:36:51.465479498Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:37:18.044672643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:37:48.046169333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:37:50.334783337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:38:18.048857933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:38:27.445169535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:47682"
level=debug ts=2026-04-29T15:38:28.337248301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:38:48.051884332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:39:07.996380299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:39:11.074777791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:47968"
ts=2026-04-29T15:39:18.053699953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:39:27.449433683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:59470"
ts=2026-04-29T15:39:48.055533723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:39:57.450962244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:40952"
level=debug ts=2026-04-29T15:40:03.786061876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:40:18.057993497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:40:48.059481423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:40:48.448858009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:40:48.450465679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:41:18.060978559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=error ts=2026-04-29T15:41:21.944991927Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:41:21.947268936Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:41:21.949495459Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:41:21.949528666Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T15:41:25.671995311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:41:27.456870328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:53892"
ts=2026-04-29T15:41:41.086871076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:44738"
ts=2026-04-29T15:41:48.063536725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:42:06.343737148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:42:18.066047366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:42:27.541868577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56306"
level=debug ts=2026-04-29T15:42:40.48996358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:42:48.070136691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:43:18.071705434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:43:18.474639407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:43:41.092022768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:42464"
ts=2026-04-29T15:43:48.073133779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:43:53.46463726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:43:53.466226346Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:44:11.092992417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:45910"
ts=2026-04-29T15:44:18.074479523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:44:27.471314631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:54070"
ts=2026-04-29T15:44:27.554716043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48810"
level=debug ts=2026-04-29T15:44:41.274862064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:44:48.077062801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:44:57.472604519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:33772"
ts=2026-04-29T15:44:57.55812477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43212"
ts=2026-04-29T15:45:11.095371069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:43206"
level=debug ts=2026-04-29T15:45:18.066254906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:45:18.07949144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:45:41.096877187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:35118"
ts=2026-04-29T15:45:48.080721638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:45:57.47527033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:48254"
ts=2026-04-29T15:46:11.098595829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:53056"
level=debug ts=2026-04-29T15:46:13.040603629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:46:18.082558938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:46:41.099464172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46184"
ts=2026-04-29T15:46:48.08365259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:46:54.528644426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:46:57.57547549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36482"
ts=2026-04-29T15:47:11.102078786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:44796"
ts=2026-04-29T15:47:18.084723055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:47:27.480378425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:52302"
ts=2026-04-29T15:47:41.103368615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:53446"
ts=2026-04-29T15:47:48.086201035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:47:55.844486456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:47:55.846083156Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:47:57.481610499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36640"
ts=2026-04-29T15:48:18.088078677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:48:45.89747528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:48:48.090713275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:48:57.588481431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48464"
ts=2026-04-29T15:49:18.094170295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:49:27.487114784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36534"
level=debug ts=2026-04-29T15:49:27.571108872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:49:48.097629792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:50:06.77177583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:50:18.100688151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:50:27.597484271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37442"
ts=2026-04-29T15:50:48.102870529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:50:57.051478669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:51:18.104116348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=error ts=2026-04-29T15:51:21.945280505Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:51:21.947255454Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:51:21.949021445Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:51:21.949043776Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T15:51:38.687548854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:51:38.68931983Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:51:41.118698285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:35342"
ts=2026-04-29T15:51:48.106281108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:51:57.607023199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42320"
level=debug ts=2026-04-29T15:52:15.068684716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:52:18.108747884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:52:27.500739494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:41140"
ts=2026-04-29T15:52:48.110457486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:52:55.355482887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:53:18.112126705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:53:27.616714483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34266"
level=debug ts=2026-04-29T15:53:40.649971608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:53:48.113780214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:53:57.507281659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:44036"
ts=2026-04-29T15:54:18.11489373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:54:22.581757796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:54:27.508846412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:46754"
ts=2026-04-29T15:54:27.623174798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48610"
ts=2026-04-29T15:54:48.117017083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:55:18.12002604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:55:21.333464669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:55:21.335138707Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:55:27.629653135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34552"
ts=2026-04-29T15:55:48.121809448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T15:55:57.516456039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:35268"
level=debug ts=2026-04-29T15:56:02.148625451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:56:18.124092555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:56:41.141134319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:49866"
ts=2026-04-29T15:56:48.127207473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T15:56:48.690899171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:57:18.129294699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T15:57:38.696963528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:57:48.131185016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:57:57.524556405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:35856"
ts=2026-04-29T15:57:57.64478803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:33516"
ts=2026-04-29T15:58:18.133426674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:58:27.526195704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:43754"
level=debug ts=2026-04-29T15:58:30.937310223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:58:48.136122288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T15:58:57.527596923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:39580"
level=debug ts=2026-04-29T15:59:06.485571629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:59:06.487381375Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:59:11.148765486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:33394"
ts=2026-04-29T15:59:18.138756445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T15:59:27.52912613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:49234"
ts=2026-04-29T15:59:41.150304012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:36742"
ts=2026-04-29T15:59:48.139762908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T15:59:49.023733217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:00:11.151867504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:34000"
ts=2026-04-29T16:00:18.140628081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:00:27.660863925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:54422"
level=debug ts=2026-04-29T16:00:36.083167019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:00:41.153591462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:37270"
ts=2026-04-29T16:00:48.142192389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:01:11.154944554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:58024"
ts=2026-04-29T16:01:18.144089871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=error ts=2026-04-29T16:01:21.942430449Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:01:21.944195276Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:01:21.945826464Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:01:21.945847381Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:01:27.667586163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59570"
level=debug ts=2026-04-29T16:01:31.498523763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:01:48.145273819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:01:57.540514998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:48216"
level=debug ts=2026-04-29T16:02:07.242153976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:02:18.14722867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:02:27.541884027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:37044"
ts=2026-04-29T16:02:41.16299257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:60760"
ts=2026-04-29T16:02:48.149127767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:02:50.98686568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:02:50.989043066Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:02:57.543025218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:50982"
ts=2026-04-29T16:03:18.150575581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:03:27.543918534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:55082"
level=debug ts=2026-04-29T16:03:37.547972899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:03:48.152662988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:03:57.54578552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:37336"
ts=2026-04-29T16:03:57.684018176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44132"
ts=2026-04-29T16:04:18.156062985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:04:33.287493245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:04:48.157697352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:05:18.159166172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:05:18.627337359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:05:27.553731997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:60104"
ts=2026-04-29T16:05:27.693747852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46942"
ts=2026-04-29T16:05:48.160181418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:05:57.696585768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34100"
level=debug ts=2026-04-29T16:06:07.934134994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:06:11.176267811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:36804"
ts=2026-04-29T16:06:18.161623043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:06:27.699546383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:42788"
ts=2026-04-29T16:06:48.164103544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:06:52.555377561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:06:52.557144431Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:07:18.16664558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:07:27.558973639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:37002"
level=debug ts=2026-04-29T16:07:39.288857294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:07:41.183147652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:42850"
ts=2026-04-29T16:07:48.167746524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:07:57.708990846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45598"
ts=2026-04-29T16:08:18.169097162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:08:23.623992263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:08:27.564484422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:44590"
ts=2026-04-29T16:08:48.170231507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:08:59.6724627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:09:18.171195499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:09:44.089885879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:09:48.172918161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:09:57.570337026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:42438"
ts=2026-04-29T16:10:18.174870404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:10:27.571939332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:38182"
ts=2026-04-29T16:10:27.724681354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34480"
level=debug ts=2026-04-29T16:10:45.872084062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:10:45.873664482Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:10:48.176164545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:10:57.572795243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:46184"
ts=2026-04-29T16:10:57.72809644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44044"
ts=2026-04-29T16:11:18.177809645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=error ts=2026-04-29T16:11:21.941459604Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:11:21.942922347Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:11:21.94445454Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:11:21.944476862Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T16:11:22.228153519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:11:27.574079544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:37852"
ts=2026-04-29T16:11:41.201806466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:55010"
ts=2026-04-29T16:11:48.179395213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:11:57.575854174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:36526"
level=debug ts=2026-04-29T16:12:14.506681969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:12:18.18046646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:12:41.204379204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:47126"
ts=2026-04-29T16:12:48.182393918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:13:01.706969033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:13:18.184787075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:13:27.583459092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:54214"
level=debug ts=2026-04-29T16:13:48.10306989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:13:48.187476544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:13:57.585370632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:58554"
ts=2026-04-29T16:14:18.189908427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:14:27.58703869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:55384"
ts=2026-04-29T16:14:27.750030102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:38346"
level=debug ts=2026-04-29T16:14:45.573099183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:14:45.575076453Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:14:48.191993106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:15:11.215244832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:54568"
ts=2026-04-29T16:15:18.193779039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:15:37.435100048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:15:48.195840662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:15:57.592094603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:52390"
ts=2026-04-29T16:16:11.216986917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:42640"
ts=2026-04-29T16:16:18.198889605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:16:27.761707281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51116"
level=debug ts=2026-04-29T16:16:30.385622459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:16:48.200624225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:16:57.595273497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:34288"
ts=2026-04-29T16:17:18.202872345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:17:26.249509314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:17:48.205909063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:18:11.225645075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:51596"
level=debug ts=2026-04-29T16:18:13.011733215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:18:18.207712832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:18:41.227214942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:37414"
ts=2026-04-29T16:18:48.208972382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:18:57.603408882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:40112"
level=debug ts=2026-04-29T16:19:00.303813473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:19:00.305376071Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:19:11.229027436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:51650"
ts=2026-04-29T16:19:18.210123071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:19:27.60438218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:35776"
level=debug ts=2026-04-29T16:19:40.307253565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:19:48.2119643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:19:57.605331168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:54360"
ts=2026-04-29T16:20:18.21472892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:20:27.786910535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44198"
level=debug ts=2026-04-29T16:20:40.164904628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:20:41.236621703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:38734"
ts=2026-04-29T16:20:48.216591588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:21:17.43540508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:21:18.218718545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=error ts=2026-04-29T16:21:21.942069736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:21:21.943946773Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:21:21.945410917Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:21:21.945431859Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:21:48.221610926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:21:57.610945651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:38050"
level=debug ts=2026-04-29T16:22:12.069990875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:22:18.224557986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:22:27.612761031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:55376"
level=debug ts=2026-04-29T16:22:46.432453704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:22:46.434082959Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:22:48.227099346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:22:57.803124822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45532"
ts=2026-04-29T16:23:18.228521814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:23:46.130995747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:23:48.229619019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:23:57.620760164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:60020"
ts=2026-04-29T16:23:57.808711643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:37554"
ts=2026-04-29T16:24:18.2304507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:24:34.887920095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:24:41.256613658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:45822"
ts=2026-04-29T16:24:48.231679453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:24:57.62295584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:40924"
ts=2026-04-29T16:25:18.233367055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:25:27.624461611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:60072"
ts=2026-04-29T16:25:27.818758951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51546"
level=debug ts=2026-04-29T16:25:34.196092576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:25:48.234518694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:26:11.262112512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:57138"
ts=2026-04-29T16:26:18.236844607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:26:23.114025991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:26:27.628336401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:43540"
ts=2026-04-29T16:26:27.825090128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41044"
ts=2026-04-29T16:26:48.240285793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:26:57.828372475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56050"
ts=2026-04-29T16:27:11.264069143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:51438"
ts=2026-04-29T16:27:18.242880457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:27:18.265290601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:27:18.266809563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:27:41.265497589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:38276"
ts=2026-04-29T16:27:48.243867868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:27:57.635450115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:47010"
level=debug ts=2026-04-29T16:28:01.046392814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:28:11.266742137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:47180"
ts=2026-04-29T16:28:18.24542133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:28:41.267709234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:38778"
level=debug ts=2026-04-29T16:28:42.510226519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:28:48.246557374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:28:57.842073725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60230"
ts=2026-04-29T16:29:11.268558252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:39680"
level=debug ts=2026-04-29T16:29:17.58490809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:29:18.247903225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:29:48.250146883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:29:57.639722207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:46400"
ts=2026-04-29T16:29:57.848337206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50474"
level=debug ts=2026-04-29T16:30:12.54551184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:30:18.25199524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:30:48.252917379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:31:08.466172256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:31:08.467849381Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:31:11.277631285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:55686"
ts=2026-04-29T16:31:18.255621449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=error ts=2026-04-29T16:31:21.94455789Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:31:21.946279407Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:31:21.947793036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:31:21.947814438Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:31:41.278700708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:56548"
ts=2026-04-29T16:31:48.257813761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:31:57.647570467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:45508"
ts=2026-04-29T16:31:57.860109882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:52674"
level=debug ts=2026-04-29T16:32:04.56086937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:32:11.280193815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:39808"
ts=2026-04-29T16:32:18.258915759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:32:27.64904664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:50256"
ts=2026-04-29T16:32:27.86363513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55666"
ts=2026-04-29T16:32:48.260552651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:33:00.320100244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:33:11.285327722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:60878"
ts=2026-04-29T16:33:18.261730447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:33:27.651481266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:48400"
level=debug ts=2026-04-29T16:33:38.598224249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:33:48.262860171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:33:57.65237322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:50914"
ts=2026-04-29T16:34:18.264457989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
level=debug ts=2026-04-29T16:34:19.917454564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:34:41.291549422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:43682"
ts=2026-04-29T16:34:48.265626261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:34:57.65559498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:51450"
ts=2026-04-29T16:34:57.884754619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59926"
level=debug ts=2026-04-29T16:35:01.412198338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:35:01.41365712Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:35:11.293055437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:33864"
ts=2026-04-29T16:35:18.267491697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:35:27.657081222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:53140"
ts=2026-04-29T16:35:27.888031491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59256"
ts=2026-04-29T16:35:48.26958283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:35:53.661195269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:35:57.65856052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:53458"
ts=2026-04-29T16:36:18.271585954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:36:42.307860057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:36:48.274469635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:37:18.276016881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:37:29.886472881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:37:41.303522488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46732"
ts=2026-04-29T16:37:48.277128382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:38:11.304960439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:49512"
level=debug ts=2026-04-29T16:38:15.117057425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:38:18.278855532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:38:48.280897196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:38:57.66963696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:56422"
level=debug ts=2026-04-29T16:39:06.092145738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:39:06.093805941Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:39:18.281954758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:39:41.309609354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:57970"
ts=2026-04-29T16:39:48.283986096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:39:55.123585148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:39:57.916593508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56850"
ts=2026-04-29T16:40:18.28591252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:40:41.312247806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:52082"
ts=2026-04-29T16:40:48.287603689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
level=debug ts=2026-04-29T16:40:50.438305043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:40:57.67520427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:57226"
ts=2026-04-29T16:41:18.289204511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=error ts=2026-04-29T16:41:21.943195445Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:41:21.944959464Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:41:21.946655425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T16:41:21.946675844Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:41:27.676487011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:54598"
ts=2026-04-29T16:41:41.314726297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:49410"
level=debug ts=2026-04-29T16:41:44.635565133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:41:48.29098052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:42:18.293062304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
ts=2026-04-29T16:42:27.932168723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51838"
level=debug ts=2026-04-29T16:42:39.952280597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:42:41.319774501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:35596"
ts=2026-04-29T16:42:48.294249241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:43:11.320975614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.44:46736"
ts=2026-04-29T16:43:18.295611066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lkftp-7d6392c2 10.130.0.44:7946"
level=debug ts=2026-04-29T16:43:41.850337726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T16:43:41.851839178Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T16:43:48.296816535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:43:57.942851025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49182"
ts=2026-04-29T16:44:18.298583676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-pxqzw-7e75e571 10.130.0.43:7946"
ts=2026-04-29T16:44:27.946381299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:59228"
level=debug ts=2026-04-29T16:44:43.65838201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T16:44:48.301316444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1a561f69 10.129.0.59:7946"
ts=2026-04-29T16:44:57.692362314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.43:49836"
ts=2026-04-29T16:44:57.949905905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:52158"