level=info ts=2026-05-01T04:08:48.445911972Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-01T04:08:48.445941541Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-01T04:08:48.447925664Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-01T04:08:48.448318967Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T04:08:48.44841557Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.87 inf=eth0
level=debug ts=2026-05-01T04:08:48.448978663Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T04:08:48.449069495Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.87 inf=eth0
level=info ts=2026-05-01T04:08:48.449521655Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-2d35dfa0
ts=2026-05-01T04:08:48.449600636Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-01T04:08:48.449618241Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.87 advertisePort=7946
level=debug ts=2026-05-01T04:08:48.449640854Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.87 advertisePort=7946
level=info ts=2026-05-01T04:08:48.449757907Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-01T04:08:48.454326629Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T04:08:48.454339438Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T04:08:48.454348269Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.592534ms
level=error ts=2026-05-01T04:08:48.454356077Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-01T04:08:48.454362554Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-01T04:08:48.456320734Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T04:08:48.45633393Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T04:08:48.467858824Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-01T04:08:48.467912269Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-01T04:08:48.467911947Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-01T04:08:48.467942587Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-01T04:08:48.467965744Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-01T04:08:48.468026449Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-01T04:08:48.468049017Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-01T04:08:48.468059963Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-01T04:08:48.468169912Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-01T04:08:48.46819349Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-01T04:08:48.468299562Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-01T04:08:48.468289337Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-01T04:08:48.468315972Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-01T04:08:48.468492347Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-01T04:08:48.468510248Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-01T04:08:48.468534637Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-01T04:08:49.469280527Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-01T04:08:49.612479167Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-01T04:08:49.612539914Z caller=loki.go:615 msg="Loki started" startup_time=1.244052719s
ts=2026-05-01T04:08:50.272854543Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T04:08:50.272879726Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:08:53.925076598Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T04:08:53.925099169Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-01T04:08:54.612827285Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-01T04:08:54.612879065Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
level=debug ts=2026-05-01T04:08:58.799081057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:09:01.171144329Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T04:09:01.17116619Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:09:13.205596037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40744"
level=debug ts=2026-05-01T04:09:16.437638903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:09:16.737283535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.87:7946"
ts=2026-05-01T04:09:16.737408298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.87:33688"
ts=2026-05-01T04:09:16.738964839Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
ts=2026-05-01T04:09:16.739614339Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.80:7946: dial tcp 10.130.0.80:7946: connect: connection refused"
ts=2026-05-01T04:09:16.739747343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.81:7946"
ts=2026-05-01T04:09:16.740136814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.84:7946"
ts=2026-05-01T04:09:16.7414672Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.86:7946: dial tcp 10.130.0.86:7946: connect: connection refused"
level=info ts=2026-05-01T04:09:16.741477635Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=28.287111214s
ts=2026-05-01T04:09:30.87261357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45390"
ts=2026-05-01T04:09:31.630775508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33514"
level=debug ts=2026-05-01T04:09:37.455600994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:09:40.615285803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:10:04.647480515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:10:10.617440167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:10:13.518159525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:38132"
ts=2026-05-01T04:10:21.432381418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54672"
ts=2026-05-01T04:10:37.180631664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52960"
level=debug ts=2026-05-01T04:10:37.468604098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T04:10:37.470270011Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:10:40.619121488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:11:07.183767254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50960"
ts=2026-05-01T04:11:10.622303102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:11:19.43725346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:11:21.435790028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60484"
ts=2026-05-01T04:11:40.624448047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:11:43.522219177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52936"
ts=2026-05-01T04:11:51.437512299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:32800"
ts=2026-05-01T04:12:07.19008885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46778"
ts=2026-05-01T04:12:10.626450498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:12:21.439116172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51676"
level=debug ts=2026-05-01T04:12:35.599522428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:12:37.193512832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58544"
ts=2026-05-01T04:12:40.62832396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:13:07.196644736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38676"
ts=2026-05-01T04:13:10.630764994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:13:13.528207128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:44584"
ts=2026-05-01T04:13:40.633313582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:13:43.530021394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:38516"
level=debug ts=2026-05-01T04:13:45.598762538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:13:51.446596193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45040"
ts=2026-05-01T04:14:10.635215463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:14:21.448286699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45382"
ts=2026-05-01T04:14:40.636672704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:14:46.459145874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T04:14:51.449973295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58730"
ts=2026-05-01T04:15:10.638376036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:15:13.535761268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56176"
ts=2026-05-01T04:15:40.640163832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:15:57.56918576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T04:15:57.570695319Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:16:10.643529767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:16:13.540462424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:48754"
ts=2026-05-01T04:16:21.456539806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53578"
ts=2026-05-01T04:16:37.217617801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49050"
ts=2026-05-01T04:16:40.645837907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:16:40.867849465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:16:43.541798746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:58860"
ts=2026-05-01T04:16:51.45811037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40142"
ts=2026-05-01T04:17:10.647656281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:17:21.459653011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59480"
level=debug ts=2026-05-01T04:17:21.91259791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:17:40.649039175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:18:07.227044465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37852"
ts=2026-05-01T04:18:10.650361975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:18:11.092170623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:18:40.651982666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=info ts=2026-05-01T04:18:54.612944098Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-01T04:18:54.61641713Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:18:54.618024176Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:18:54.619572818Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:18:54.619595602Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T04:18:54.619607425Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-01T04:18:59.667135593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:19:10.653789781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:19:37.235783509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58942"
ts=2026-05-01T04:19:40.656607892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:19:46.31525288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:19:46.316692283Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:20:07.239194346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51682"
ts=2026-05-01T04:20:10.658423527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:20:13.557679417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54236"
ts=2026-05-01T04:20:21.473465309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33440"
level=debug ts=2026-05-01T04:20:22.938188185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:20:40.659635674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:20:57.915366748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:21:10.661448654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:21:21.477861153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42340"
ts=2026-05-01T04:21:40.663210697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:21:47.317192826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:22:07.250477918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43184"
ts=2026-05-01T04:22:10.665356653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:22:13.567301817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43394"
ts=2026-05-01T04:22:37.253991972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44522"
level=debug ts=2026-05-01T04:22:39.281408691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:22:40.666590852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:22:43.568532132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51894"
ts=2026-05-01T04:22:51.482556393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36220"
ts=2026-05-01T04:23:10.66809331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:23:13.569648903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:44364"
level=debug ts=2026-05-01T04:23:30.593001852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:23:30.59475265Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:23:37.259710071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59466"
ts=2026-05-01T04:23:40.669995621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:23:43.571595495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:42280"
ts=2026-05-01T04:24:10.671704029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:24:15.143346007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:24:37.266020896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34900"
ts=2026-05-01T04:24:40.673248391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:24:52.11342759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:25:10.675310663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:25:27.672590703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:25:37.272399833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40588"
ts=2026-05-01T04:25:40.677197328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:26:10.679062069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:26:12.694015564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:26:21.499896877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57008"
ts=2026-05-01T04:26:37.278393857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60234"
ts=2026-05-01T04:26:40.68068902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:26:43.586988043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46316"
level=debug ts=2026-05-01T04:27:04.003626621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:27:04.005405126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:27:10.683388626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:27:13.588181011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:38462"
ts=2026-05-01T04:27:21.50416169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44114"
level=debug ts=2026-05-01T04:27:38.183717978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:27:40.686674939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:27:43.589197553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:55676"
ts=2026-05-01T04:28:10.689409218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:28:21.507218601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34812"
level=debug ts=2026-05-01T04:28:37.714488046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:28:40.690841554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=error ts=2026-05-01T04:28:54.623898189Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:28:54.625795754Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:28:54.627625528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:28:54.627652736Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:29:07.29389598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51046"
ts=2026-05-01T04:29:10.693038972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:29:21.511927697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45352"
level=debug ts=2026-05-01T04:29:23.014290368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:29:37.297191015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36622"
ts=2026-05-01T04:29:40.696661183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:30:07.300213609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56252"
ts=2026-05-01T04:30:10.699590999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:30:13.598319088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56282"
level=debug ts=2026-05-01T04:30:24.360213821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:30:40.702202028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:31:10.500382015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:31:10.502867208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:31:10.703805811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:31:40.705164728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:31:46.029211238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:32:10.707552811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:32:27.701562341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:32:40.709555928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:32:43.632168885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:57182"
level=debug ts=2026-05-01T04:33:08.395042431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:33:10.713143761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:33:40.714541299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:33:51.527589532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37958"
level=debug ts=2026-05-01T04:34:03.512864254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:34:07.328650853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53316"
ts=2026-05-01T04:34:10.717295393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:34:40.720597377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:34:43.63999679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40538"
ts=2026-05-01T04:34:51.532753898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45484"
level=debug ts=2026-05-01T04:35:05.69866156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:35:05.700314193Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:35:10.722695302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:35:13.641468608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46056"
ts=2026-05-01T04:35:21.533730316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36602"
ts=2026-05-01T04:35:40.723913352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:36:03.846724117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:36:10.732349413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:36:40.737580314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:36:48.555364461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:37:07.347412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42660"
ts=2026-05-01T04:37:10.738914264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:37:28.25551776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:37:37.35003867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52924"
ts=2026-05-01T04:37:40.741457561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:37:43.649802531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:34766"
ts=2026-05-01T04:37:51.542683576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42484"
level=debug ts=2026-05-01T04:38:09.469319171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:38:10.744033818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:38:13.650556652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33662"
ts=2026-05-01T04:38:40.745860589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:38:51.545118144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39624"
level=debug ts=2026-05-01T04:38:51.879309391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:38:51.881128292Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:38:54.622519447Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:38:54.624343991Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:38:54.626250794Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:38:54.626274902Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:39:10.747540624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:39:37.362209727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53250"
level=debug ts=2026-05-01T04:39:39.22045276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:39:40.749292946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:40:10.751792538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:40:21.552794471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60000"
level=debug ts=2026-05-01T04:40:37.36587272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:40:40.753839351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:41:10.75484144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:41:27.099828954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:41:40.756130881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:41:51.560820508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48970"
ts=2026-05-01T04:42:07.376949152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54590"
level=debug ts=2026-05-01T04:42:09.880700954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:42:10.758137278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:42:40.759797765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:43:07.383052517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51428"
level=debug ts=2026-05-01T04:43:08.763089844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:43:08.764567735Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:43:10.762270032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:43:40.76498579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:43:47.387639477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:44:10.765854496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:44:27.895478971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:44:40.767363804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:45:10.769094583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:45:16.662594325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:45:40.771375272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:45:51.576085992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55086"
level=debug ts=2026-05-01T04:46:00.834583946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:46:07.403201787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46922"
ts=2026-05-01T04:46:10.773690435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:46:37.40572343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42838"
ts=2026-05-01T04:46:40.775744613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:46:43.412251445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:46:43.41374221Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:47:10.778532893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:47:21.583783958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58592"
level=debug ts=2026-05-01T04:47:23.948796702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:47:40.781170689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:47:43.693295319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43448"
ts=2026-05-01T04:47:51.591395446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48864"
ts=2026-05-01T04:48:10.783691329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:48:15.541702703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:48:21.59319372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44814"
ts=2026-05-01T04:48:37.417378636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44278"
ts=2026-05-01T04:48:40.790384291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:48:51.595028486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56046"
level=error ts=2026-05-01T04:48:54.624331461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:48:54.626305068Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:48:54.62809081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:48:54.628115673Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T04:48:59.741630946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:49:10.792389928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:49:40.793784948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:50:00.621839917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:50:10.795520886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:50:21.603720269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45262"
level=debug ts=2026-05-01T04:50:36.316749708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:50:36.318435046Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:50:40.797917393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:50:43.710397657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:55572"
ts=2026-05-01T04:51:10.799809715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:51:13.712152553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:37248"
level=debug ts=2026-05-01T04:51:18.724682877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:51:37.43547314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52266"
ts=2026-05-01T04:51:40.804588884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T04:52:06.636746044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:52:07.439015281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59424"
ts=2026-05-01T04:52:10.808042576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:52:40.812607529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:53:01.69290314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:53:10.815858924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:53:21.61889681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47894"
ts=2026-05-01T04:53:40.817429212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:53:43.728589833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50582"
level=debug ts=2026-05-01T04:53:44.811933194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:53:51.620517213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43276"
ts=2026-05-01T04:54:07.450595642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36668"
ts=2026-05-01T04:54:10.818916297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:54:13.730283537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:59132"
ts=2026-05-01T04:54:21.621621248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37442"
ts=2026-05-01T04:54:37.453850859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40492"
ts=2026-05-01T04:54:40.821638886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:54:41.551299168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:54:41.553091792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:54:51.622911029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59208"
ts=2026-05-01T04:55:10.824906807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:55:27.208935119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:55:37.460713793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57618"
ts=2026-05-01T04:55:40.828267714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:55:51.624739968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51420"
ts=2026-05-01T04:56:10.830692602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T04:56:13.736645844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52902"
level=debug ts=2026-05-01T04:56:16.593084807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:56:21.626157962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50426"
ts=2026-05-01T04:56:40.831802497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:56:54.304272498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:57:07.469951351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33108"
ts=2026-05-01T04:57:10.834218954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T04:57:13.738871392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:58138"
ts=2026-05-01T04:57:21.630674947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42200"
ts=2026-05-01T04:57:37.472677573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53620"
ts=2026-05-01T04:57:40.835833961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T04:57:41.65785028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:57:51.631878276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47614"
ts=2026-05-01T04:58:10.838332999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:58:16.983862366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:58:16.985633672Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:58:40.841161863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T04:58:52.094799403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T04:58:54.622402831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:58:54.623934457Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:58:54.625611214Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T04:58:54.625633912Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T04:59:10.843508574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:59:13.744607908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:32902"
level=debug ts=2026-05-01T04:59:33.114026221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T04:59:40.844969096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T04:59:43.746121986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:42784"
ts=2026-05-01T04:59:51.638053201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58092"
ts=2026-05-01T05:00:10.846013877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:00:11.936407152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:00:21.639509854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59804"
ts=2026-05-01T05:00:37.500370673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53480"
ts=2026-05-01T05:00:40.847742078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:01:10.849468858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:01:13.490318993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:01:13.751551902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:34872"
ts=2026-05-01T05:01:40.851885233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:01:43.75273132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49660"
level=debug ts=2026-05-01T05:01:56.200688062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:01:56.202392382Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:02:10.853692488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:02:34.651277738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:02:37.513070988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51958"
ts=2026-05-01T05:02:40.856057031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:02:43.757104186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33658"
ts=2026-05-01T05:02:51.651479545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58120"
ts=2026-05-01T05:03:10.859083899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:03:11.752290368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:03:40.861012346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:03:43.759317438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45010"
level=debug ts=2026-05-01T05:03:55.430508287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:04:10.862920699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:04:40.865179685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:04:43.762461111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:53340"
level=debug ts=2026-05-01T05:04:44.055685855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:05:10.867510233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:05:13.764330484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:36974"
ts=2026-05-01T05:05:21.662368335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41132"
level=debug ts=2026-05-01T05:05:21.8646607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:05:21.866514935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:05:40.869131606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:05:43.765788484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51528"
ts=2026-05-01T05:05:51.663952173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45440"
level=debug ts=2026-05-01T05:06:10.860362208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:06:10.870575788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:06:13.76693356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:58098"
ts=2026-05-01T05:06:37.537171009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57124"
ts=2026-05-01T05:06:40.872833486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:06:51.668653105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58932"
level=debug ts=2026-05-01T05:07:06.084136508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:07:10.874783108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:07:13.771477388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46484"
ts=2026-05-01T05:07:40.87594357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:07:43.772662766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52262"
level=debug ts=2026-05-01T05:08:03.272658073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:08:07.545418606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59256"
ts=2026-05-01T05:08:10.877291542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:08:40.878455224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=error ts=2026-05-01T05:08:54.623766902Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:08:54.62541479Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:08:54.626905515Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:08:54.626926641Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-01T05:09:05.326020861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:09:10.880587872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:09:21.675672536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39038"
ts=2026-05-01T05:09:37.554096451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58136"
ts=2026-05-01T05:09:40.882815471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:09:45.20860465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:09:45.210026295Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:10:10.88444791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:10:21.67781369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52768"
ts=2026-05-01T05:10:37.560316291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56862"
ts=2026-05-01T05:10:40.886195058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:10:41.406949087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:11:10.887621344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:11:13.784987473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50368"
level=debug ts=2026-05-01T05:11:31.370739188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:11:40.888465946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:11:43.785945141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:53236"
ts=2026-05-01T05:11:51.68433116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38876"
ts=2026-05-01T05:12:10.890253705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:12:13.787478648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:42754"
level=debug ts=2026-05-01T05:12:20.020321519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:12:40.89171407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:12:59.566633072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:13:07.57477392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44830"
ts=2026-05-01T05:13:10.893163121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:13:37.578178311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41672"
level=debug ts=2026-05-01T05:13:40.104751346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:13:40.106381906Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:13:40.894616082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:14:07.581316328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49458"
ts=2026-05-01T05:14:10.895836114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:14:21.698343381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36464"
level=debug ts=2026-05-01T05:14:22.966437644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:14:40.898038893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:14:58.723234692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:15:07.587079363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51944"
ts=2026-05-01T05:15:10.899219246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:15:13.799920578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45078"
level=debug ts=2026-05-01T05:15:39.878102535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:15:40.901768523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:16:07.592675655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39506"
ts=2026-05-01T05:16:10.904294804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:16:13.803166919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51130"
ts=2026-05-01T05:16:21.707854544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38876"
ts=2026-05-01T05:16:37.595912165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45228"
ts=2026-05-01T05:16:40.906262144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:16:41.319577204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:17:10.907213715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:17:21.712343179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40874"
level=debug ts=2026-05-01T05:17:25.18045404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:17:25.181999154Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:17:40.909086856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:18:07.605208819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58616"
ts=2026-05-01T05:18:10.910996347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:18:13.885742141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:18:37.607915952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59352"
ts=2026-05-01T05:18:40.912252547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:18:43.812939892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:34394"
level=debug ts=2026-05-01T05:18:49.386044679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:18:51.719608171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49072"
level=error ts=2026-05-01T05:18:54.623693506Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:18:54.625336203Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:18:54.626804833Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:18:54.626823248Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:19:10.913988538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:19:21.720761731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37530"
level=debug ts=2026-05-01T05:19:38.005729156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:19:40.915254088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:20:07.617294549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43912"
ts=2026-05-01T05:20:10.91677341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:20:34.03814819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:20:37.62047138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55472"
ts=2026-05-01T05:20:40.919155631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:21:07.623668168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40028"
ts=2026-05-01T05:21:10.921944693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:21:21.726507377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:40248"
level=debug ts=2026-05-01T05:21:29.103110155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:21:29.104772926Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:21:40.923614335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:22:07.629388598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42030"
level=debug ts=2026-05-01T05:22:09.728545606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:22:10.925920454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:22:21.730691202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58812"
ts=2026-05-01T05:22:37.632866867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47470"
ts=2026-05-01T05:22:40.929559556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:22:51.73164896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59032"
level=debug ts=2026-05-01T05:22:52.046178265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:23:10.931145553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:23:21.732795704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39504"
ts=2026-05-01T05:23:40.933537583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:23:43.836319953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50148"
level=debug ts=2026-05-01T05:23:50.343248951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:24:10.935283408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:24:40.936291635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:24:47.121106022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:24:51.738451614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50210"
ts=2026-05-01T05:25:07.648293634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58258"
ts=2026-05-01T05:25:10.938629268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:25:13.842452789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:47728"
level=debug ts=2026-05-01T05:25:34.553173856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:25:34.554743599Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:25:40.940434843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:25:43.843875455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49228"
ts=2026-05-01T05:26:10.942286173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:26:35.415210302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:26:37.65786059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55038"
ts=2026-05-01T05:26:40.944300521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:26:43.846493739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35694"
ts=2026-05-01T05:27:07.660613433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45584"
ts=2026-05-01T05:27:10.946242532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:27:37.590333195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:27:37.663852225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41552"
ts=2026-05-01T05:27:40.948623411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:27:51.750295033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33070"
ts=2026-05-01T05:28:07.667041533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43592"
ts=2026-05-01T05:28:10.950431473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:28:19.000670784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:28:40.952249476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=error ts=2026-05-01T05:28:54.623469993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:28:54.625281253Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:28:54.626897091Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:28:54.626917105Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:29:07.673159552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39166"
ts=2026-05-01T05:29:10.954032219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:29:12.631100441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:29:40.955578167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:30:02.140119349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:30:02.141777846Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:30:07.678862729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44034"
ts=2026-05-01T05:30:10.95696177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:30:40.958466321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:30:45.723320326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:30:51.762764045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48272"
ts=2026-05-01T05:31:07.685079151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41348"
ts=2026-05-01T05:31:10.960381763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:31:13.860191013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:38822"
ts=2026-05-01T05:31:40.962471082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:31:42.024666411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:32:10.964188441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=debug ts=2026-05-01T05:32:33.121517143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:32:37.69420288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56736"
ts=2026-05-01T05:32:40.965501849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:32:43.865398054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54872"
ts=2026-05-01T05:33:10.967320022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:33:13.866284979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33372"
level=debug ts=2026-05-01T05:33:31.067511564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:33:40.968860434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:33:43.86709802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:36600"
ts=2026-05-01T05:34:10.970322922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:34:21.77937168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51010"
level=debug ts=2026-05-01T05:34:27.12085013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:34:27.122523103Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:34:40.972283696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:34:43.869701637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39594"
ts=2026-05-01T05:35:10.974586786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:35:13.666331544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:35:13.870857014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:44492"
ts=2026-05-01T05:35:37.714085074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53920"
ts=2026-05-01T05:35:40.976819476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:35:57.993565528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:36:07.717290338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58514"
ts=2026-05-01T05:36:10.979657157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:36:37.720219492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56010"
ts=2026-05-01T05:36:40.981293012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:36:43.875448897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:53568"
level=debug ts=2026-05-01T05:36:55.770707807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:37:10.983205905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:37:37.726493062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59246"
ts=2026-05-01T05:37:40.985171112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:37:46.768048503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:38:10.987733495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:38:13.883068937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33722"
level=debug ts=2026-05-01T05:38:29.030957499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:38:29.032851051Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:38:40.989886575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
level=error ts=2026-05-01T05:38:54.624746921Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:38:54.626468959Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:38:54.627972086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:38:54.62799156Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:39:10.991040236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:39:13.886786764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:58386"
level=debug ts=2026-05-01T05:39:17.922445283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:39:37.739066435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41048"
ts=2026-05-01T05:39:40.992734695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:39:43.887708003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54666"
ts=2026-05-01T05:40:10.994307729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:40:13.888692533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51700"
level=debug ts=2026-05-01T05:40:15.716612596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:40:40.996758776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:40:51.803706284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52686"
level=debug ts=2026-05-01T05:41:06.922471508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:41:10.997744108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:41:13.892103746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:41424"
ts=2026-05-01T05:41:40.999504842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:41:43.893792136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56994"
level=debug ts=2026-05-01T05:41:48.020284799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:41:51.808239803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44822"
ts=2026-05-01T05:42:07.754000508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49108"
ts=2026-05-01T05:42:11.001371665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:42:28.702783662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:42:28.704297639Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:42:41.002730626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:42:51.812398176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57590"
ts=2026-05-01T05:43:07.760016977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53402"
ts=2026-05-01T05:43:11.004371703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:43:13.89779858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51096"
level=debug ts=2026-05-01T05:43:15.107886024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:43:37.763412697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36152"
ts=2026-05-01T05:43:41.005760678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:43:43.898610021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:44080"
level=debug ts=2026-05-01T05:43:52.799633929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:44:07.766336192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54170"
ts=2026-05-01T05:44:11.007077157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
ts=2026-05-01T05:44:13.900171661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:48680"
level=debug ts=2026-05-01T05:44:34.623966403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:44:41.00832139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:44:43.901542836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:55072"
ts=2026-05-01T05:44:51.819331718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:51328"
ts=2026-05-01T05:45:11.009638378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:45:14.098074807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:45:21.820179856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41192"
ts=2026-05-01T05:45:41.01129616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f8hpm-a1e9104f 10.130.0.81:7946"
level=debug ts=2026-05-01T05:45:51.084248729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T05:45:51.085998021Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:45:51.822017806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48970"
ts=2026-05-01T05:46:11.012885087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:46:41.015320917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:46:41.637787362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:47:07.784455189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59978"
ts=2026-05-01T05:47:11.018847375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:47:39.305606609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:47:41.022154936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
ts=2026-05-01T05:47:43.917471656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43088"
ts=2026-05-01T05:48:07.790635311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59554"
ts=2026-05-01T05:48:11.0260439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"
level=debug ts=2026-05-01T05:48:36.658394808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T05:48:41.02824505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-g2cnx-8d703f12 10.130.0.84:7946"
ts=2026-05-01T05:48:43.920465963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:37282"
ts=2026-05-01T05:48:51.83856415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36802"
level=error ts=2026-05-01T05:48:54.623775191Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:48:54.625769436Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:48:54.627416284Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T05:48:54.627439716Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T05:49:11.030050952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c49266fe 10.128.0.79:7946"