level=info ts=2026-05-14T07:11:11.968050343Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-14T07:11:11.968106733Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-14T07:11:11.969938752Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-14T07:11:11.970358384Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T07:11:11.97044712Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.84 inf=eth0
level=debug ts=2026-05-14T07:11:11.971275007Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-14T07:11:11.971358867Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.84 inf=eth0
level=info ts=2026-05-14T07:11:11.971464909Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-889214fd
ts=2026-05-14T07:11:11.971560805Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-14T07:11:11.971573014Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.84 advertisePort=7946
level=debug ts=2026-05-14T07:11:11.971592825Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.84 advertisePort=7946
level=info ts=2026-05-14T07:11:11.971735833Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-14T07:11:11.978217159Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T07:11:11.978239964Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-14T07:11:11.978253969Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.520344ms
level=error ts=2026-05-14T07:11:11.978266806Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-14T07:11:11.97828451Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-14T07:11:11.981205472Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T07:11:11.981222223Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-14T07:11:11.988253213Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-14T07:11:11.988656062Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-14T07:11:11.988680619Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-14T07:11:11.988778444Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-14T07:11:11.989134856Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-14T07:11:11.989133373Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-14T07:11:11.989146656Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-14T07:11:11.989217515Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-14T07:11:11.989234744Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-14T07:11:11.989239209Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-14T07:11:11.989301928Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-14T07:11:11.989325812Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-14T07:11:11.989341194Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-14T07:11:11.989483579Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-14T07:11:11.989493719Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-14T07:11:11.989503768Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-14T07:11:12.989943898Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-14T07:11:13.148196427Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-14T07:11:13.148256167Z caller=loki.go:615 msg="Loki started" startup_time=1.253573666s
ts=2026-05-14T07:11:13.887832371Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T07:11:13.88786556Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:11:17.687384244Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T07:11:17.687407498Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-14T07:11:18.149223616Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-14T07:11:18.149300454Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-14T07:11:22.265844625Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-14T07:11:22.265873026Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-14T07:11:25.769258733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:11:30.014179766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58822"
ts=2026-05-14T07:11:31.991947728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.84:7946"
ts=2026-05-14T07:11:31.992055994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43576"
ts=2026-05-14T07:11:31.99377382Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-05-14T07:11:31.994253398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
level=info ts=2026-05-14T07:11:31.994958047Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.016666187s
ts=2026-05-14T07:11:35.215920781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47284"
ts=2026-05-14T07:11:35.511388791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43944"
level=debug ts=2026-05-14T07:11:41.034184325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:11:47.478684232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45746"
ts=2026-05-14T07:11:48.495320174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:11:53.069188385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52880"
level=debug ts=2026-05-14T07:11:59.795682563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:12:02.098915746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49122"
ts=2026-05-14T07:12:17.481593612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59470"
ts=2026-05-14T07:12:18.497401042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:12:23.070893451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54818"
level=debug ts=2026-05-14T07:12:23.68219694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:12:47.484914725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34818"
ts=2026-05-14T07:12:48.500522205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:12:53.072692854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60612"
level=debug ts=2026-05-14T07:12:57.653549102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T07:12:57.655323166Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:13:18.502746687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:13:30.330829501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:13:47.490999968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41792"
ts=2026-05-14T07:13:48.504791546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:14:18.50672267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:14:28.719112043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:14:47.497104247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44116"
ts=2026-05-14T07:14:48.508737145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:14:53.083036639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55416"
ts=2026-05-14T07:15:17.500152062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57932"
ts=2026-05-14T07:15:18.511192957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:15:23.143136739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:15:32.117130112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50748"
ts=2026-05-14T07:15:48.513536954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:15:53.087176828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49306"
ts=2026-05-14T07:16:17.506793777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44100"
ts=2026-05-14T07:16:18.515765203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:16:18.566222926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-14T07:16:48.517790946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:16:53.091429581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60488"
ts=2026-05-14T07:17:17.513160488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42628"
ts=2026-05-14T07:17:18.518734433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:17:25.100079213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-14T07:17:25.102048952Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:17:32.123906015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57516"
ts=2026-05-14T07:17:48.520119575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:18:02.125007789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42436"
ts=2026-05-14T07:18:18.522317846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:18:21.124554808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:18:23.098994331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46984"
ts=2026-05-14T07:18:47.522546341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41336"
ts=2026-05-14T07:18:48.524446705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:19:00.109693279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:19:17.52528903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42314"
ts=2026-05-14T07:19:18.527706643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:19:47.527971564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34800"
ts=2026-05-14T07:19:48.530705471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:19:53.10642743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41544"
level=debug ts=2026-05-14T07:20:01.192958886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:20:02.135201447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:57184"
ts=2026-05-14T07:20:18.532819536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:20:36.712322231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:20:47.533389349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57502"
ts=2026-05-14T07:20:48.535798702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:20:53.110460062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46548"
level=info ts=2026-05-14T07:21:18.149818572Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-14T07:21:18.15448072Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:21:18.157136695Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:21:18.159910421Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:21:18.159963923Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-14T07:21:18.159980676Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-14T07:21:18.538359694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:21:23.112668352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42030"
level=debug ts=2026-05-14T07:21:36.656798793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:21:36.658714065Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:21:47.539213396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37610"
ts=2026-05-14T07:21:48.540163818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:22:18.542243793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:22:32.147726818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43948"
level=debug ts=2026-05-14T07:22:33.830389719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:22:48.545086093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:22:53.12001561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53354"
level=debug ts=2026-05-14T07:23:10.187501926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:23:17.548767317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55188"
ts=2026-05-14T07:23:18.547516708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:23:48.550537587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:23:54.542555133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:24:18.554349207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:24:32.157952027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:51974"
ts=2026-05-14T07:24:47.561149639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42840"
ts=2026-05-14T07:24:48.557658748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:24:53.129587498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52906"
level=debug ts=2026-05-14T07:24:53.821851229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:25:17.564750016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42558"
ts=2026-05-14T07:25:18.560599048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:25:48.056468686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:25:48.05808059Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:25:48.565321819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:26:18.566867858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:26:23.135516311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34178"
ts=2026-05-14T07:26:32.172540628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46536"
level=debug ts=2026-05-14T07:26:35.655025065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:26:48.569771504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:26:53.137968995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34148"
ts=2026-05-14T07:27:02.174735677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34140"
ts=2026-05-14T07:27:18.574680663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:27:36.988638664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:27:48.578467049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:28:18.580480773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:28:23.151406754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59050"
level=debug ts=2026-05-14T07:28:34.876272897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:28:47.58786757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51612"
ts=2026-05-14T07:28:48.58277297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:28:53.15476251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53788"
ts=2026-05-14T07:29:18.584794092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:29:33.569773526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:29:48.585974788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:29:53.159709153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58108"
ts=2026-05-14T07:30:17.596972185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60042"
ts=2026-05-14T07:30:18.587535031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:30:29.684879085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:30:29.686794159Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:30:47.600196362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38770"
ts=2026-05-14T07:30:48.589121885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:31:08.039187106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:31:17.603294916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37808"
level=error ts=2026-05-14T07:31:18.164328943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:31:18.166786598Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:31:18.168936323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:31:18.168959196Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:31:18.591411101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:31:23.168676849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41332"
ts=2026-05-14T07:31:48.594322939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:31:51.785545783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:32:02.200698101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38216"
ts=2026-05-14T07:32:18.596333595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:32:23.174005562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45816"
ts=2026-05-14T07:32:48.599218969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:32:53.14431926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:32:53.176302201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34964"
ts=2026-05-14T07:33:17.615669324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38706"
ts=2026-05-14T07:33:18.602241106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:33:30.171069984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:33:47.619304431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36986"
ts=2026-05-14T07:33:48.603808966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:34:18.606217606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:34:31.901397706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:34:31.903329683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:34:47.625347635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39888"
ts=2026-05-14T07:34:48.608630511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:35:02.21727851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58228"
ts=2026-05-14T07:35:18.609885679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:35:29.162369211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:35:47.631236671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34154"
ts=2026-05-14T07:35:48.611879812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:35:53.191623344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42926"
level=debug ts=2026-05-14T07:36:04.081466174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:36:18.614104429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:36:40.230286714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:36:48.616438121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:36:53.197076429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46290"
ts=2026-05-14T07:37:17.640775091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46956"
ts=2026-05-14T07:37:18.619429549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:37:23.19963552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48956"
level=debug ts=2026-05-14T07:37:28.120849934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:37:47.644443591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37912"
ts=2026-05-14T07:37:48.622058055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:37:53.201318796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49560"
ts=2026-05-14T07:38:18.623155245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:38:23.203173919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37776"
level=debug ts=2026-05-14T07:38:26.509496052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:38:26.511160841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:38:48.625194278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:39:02.238922487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34884"
level=debug ts=2026-05-14T07:39:04.761049368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:39:18.627204593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:39:47.661114951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36268"
ts=2026-05-14T07:39:48.629078769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:39:53.210589784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42266"
level=debug ts=2026-05-14T07:40:00.323226774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:40:02.242750285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34176"
ts=2026-05-14T07:40:18.631529602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:40:23.212990083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34612"
level=debug ts=2026-05-14T07:40:35.040946706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:40:47.667813635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54904"
ts=2026-05-14T07:40:48.634076362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:41:16.939539908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:41:17.670860889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54156"
level=error ts=2026-05-14T07:41:18.164984565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:41:18.166742899Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:41:18.168392171Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:41:18.168423231Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:41:18.636795572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:41:23.217874775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51132"
ts=2026-05-14T07:41:47.673939387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35160"
ts=2026-05-14T07:41:48.640314152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:42:17.67757568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38884"
level=debug ts=2026-05-14T07:42:18.080924161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:42:18.08265903Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:42:18.643307901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:42:47.680507653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58124"
ts=2026-05-14T07:42:48.646008524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:42:56.378370494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:43:18.648239341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:43:45.598305867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:43:47.687497559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60410"
ts=2026-05-14T07:43:48.650390694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:44:02.263460207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58280"
ts=2026-05-14T07:44:18.651699049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:44:32.264827145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60380"
level=debug ts=2026-05-14T07:44:35.805149804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:44:48.653728761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:45:18.655559568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:45:30.964492315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:45:47.700448564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42318"
ts=2026-05-14T07:45:48.658434357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:45:53.240040996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34650"
ts=2026-05-14T07:46:02.27142933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53910"
ts=2026-05-14T07:46:18.661593276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:46:19.151767236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:46:19.153471719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:46:32.272884263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:60050"
ts=2026-05-14T07:46:48.664807811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:47:02.274212531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:32974"
ts=2026-05-14T07:47:18.667983687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:47:21.524554548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:47:48.670944456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:48:13.361156059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:48:18.673114577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:48:23.250051005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37098"
ts=2026-05-14T07:48:47.718996525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34594"
ts=2026-05-14T07:48:48.67539385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:48:56.334841957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:49:18.676811769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:49:23.255075088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37300"
ts=2026-05-14T07:49:32.285940648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34342"
level=debug ts=2026-05-14T07:49:37.378301637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:49:47.726003998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34996"
ts=2026-05-14T07:49:48.679217858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:50:02.287625759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:55134"
ts=2026-05-14T07:50:18.682652295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:50:20.806208386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:50:20.807897805Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:50:48.685797061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:51:02.290835669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43904"
level=debug ts=2026-05-14T07:51:06.108494068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:51:17.735741213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40676"
level=error ts=2026-05-14T07:51:18.165707304Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:51:18.16753056Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:51:18.169567344Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T07:51:18.169589161Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:51:18.688459199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:51:23.265157396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34714"
ts=2026-05-14T07:51:48.690966178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T07:52:01.637364338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:52:18.694712931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:52:23.270933995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42384"
ts=2026-05-14T07:52:48.697238155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:53:00.404987993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:53:18.69894809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:53:23.275901322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46032"
ts=2026-05-14T07:53:48.700851711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:53:57.082063217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:54:18.702626518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T07:54:44.085183259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:54:44.087014647Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:54:48.704719526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:55:02.310729461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56296"
ts=2026-05-14T07:55:18.707099567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:55:41.895595719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:55:48.708451889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:56:02.31433154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54958"
ts=2026-05-14T07:56:18.710471827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:56:23.28972357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41410"
level=debug ts=2026-05-14T07:56:24.386999111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:56:32.315397263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53254"
ts=2026-05-14T07:56:48.711842338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T07:57:14.192674336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:57:17.77266608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55496"
ts=2026-05-14T07:57:18.712940998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:57:32.319061498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45212"
ts=2026-05-14T07:57:48.714408296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:57:53.296821132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44772"
level=debug ts=2026-05-14T07:57:54.316888672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:58:18.715741472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T07:58:32.323089398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56528"
level=debug ts=2026-05-14T07:58:45.71873287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T07:58:45.720431538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T07:58:47.782378661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48582"
ts=2026-05-14T07:58:48.718341527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T07:58:53.300685422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50598"
ts=2026-05-14T07:59:02.32478816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:37722"
ts=2026-05-14T07:59:18.720673527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:59:32.326517274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58256"
level=debug ts=2026-05-14T07:59:36.434118847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T07:59:48.722419185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T07:59:53.306246672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42418"
ts=2026-05-14T08:00:18.724344738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:00:19.73625621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:00:48.726033728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:00:53.311474877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42312"
level=debug ts=2026-05-14T08:01:09.584471688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:01:18.163438797Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:01:18.165390835Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:01:18.16734843Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:01:18.16736965Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:01:18.729649083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:01:48.731953592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:02:02.338879211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45976"
level=debug ts=2026-05-14T08:02:07.86070377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:02:18.734298134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:02:48.73634534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:02:59.344706535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:02:59.346307985Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:03:18.737807261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:03:47.812092265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38462"
ts=2026-05-14T08:03:48.739174856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:03:51.653685619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:03:53.329882104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37548"
ts=2026-05-14T08:04:17.814957099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43712"
ts=2026-05-14T08:04:18.741785341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:04:44.142742576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:04:47.817725929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36672"
ts=2026-05-14T08:04:48.744916254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:04:53.335527587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48242"
ts=2026-05-14T08:05:02.355007717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36902"
ts=2026-05-14T08:05:18.748338056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:05:27.431636635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:05:32.35684896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33656"
ts=2026-05-14T08:05:48.750716484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:06:18.752829864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:06:25.631788013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:06:47.829498809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34456"
ts=2026-05-14T08:06:48.755911514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:07:08.46378242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:07:08.466700439Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:07:18.757988603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:07:48.760074463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:07:53.351162518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57588"
level=debug ts=2026-05-14T08:08:00.241957047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:08:02.370340559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38688"
ts=2026-05-14T08:08:18.762291296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:08:39.586764257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:08:47.841286498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54882"
ts=2026-05-14T08:08:48.764914502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:08:53.356191588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55326"
ts=2026-05-14T08:09:02.375377981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:48386"
level=debug ts=2026-05-14T08:09:16.912110196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:09:17.844600191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46668"
ts=2026-05-14T08:09:18.766822141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:09:32.377514138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:52278"
ts=2026-05-14T08:09:48.768015625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:09:53.360925556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59412"
level=debug ts=2026-05-14T08:10:10.797784621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:10:17.850818356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53752"
ts=2026-05-14T08:10:18.769727811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:10:32.381821603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49092"
ts=2026-05-14T08:10:48.771559478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:10:56.926659438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:10:56.929196791Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:11:02.383328271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:43782"
level=error ts=2026-05-14T08:11:18.164285482Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:11:18.16619227Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:11:18.167991036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:11:18.168016824Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:11:18.773472359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:11:47.861337155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41214"
level=debug ts=2026-05-14T08:11:48.138490006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:11:48.774818175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:11:53.373581811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50904"
ts=2026-05-14T08:12:17.864427684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35188"
ts=2026-05-14T08:12:18.776991686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:12:29.256584603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:12:48.78033921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:13:18.252042669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:13:18.783146219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:13:23.380439961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50332"
ts=2026-05-14T08:13:32.395722264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:49498"
ts=2026-05-14T08:13:48.78418336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:14:02.397780041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53302"
level=debug ts=2026-05-14T08:14:03.765799841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:14:18.786876842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:14:46.167079928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:14:46.169646896Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:14:48.789068065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:15:02.402153218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:46276"
ts=2026-05-14T08:15:17.883103439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56174"
ts=2026-05-14T08:15:18.790404913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:15:23.067279702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:15:23.39292992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60222"
ts=2026-05-14T08:15:48.792288371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:15:53.395480767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54348"
level=debug ts=2026-05-14T08:15:59.826201183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:16:18.795491639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:16:23.39772648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54764"
ts=2026-05-14T08:16:47.891336261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43444"
ts=2026-05-14T08:16:48.798558745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:16:53.211032393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:17:18.800449726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:17:32.414823117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56626"
ts=2026-05-14T08:17:48.802784608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:17:52.243507775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:18:18.806363013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:18:36.321267059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:18:36.32306086Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:18:48.809657928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:19:17.906498701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57430"
ts=2026-05-14T08:19:18.812184695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:19:19.38341001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:19:47.909594155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52992"
ts=2026-05-14T08:19:48.813903459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:19:58.41686495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:20:18.814754826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:20:32.429235274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50768"
level=debug ts=2026-05-14T08:20:40.884187909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:20:47.915667735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52234"
ts=2026-05-14T08:20:48.815719196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:20:53.424477092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47824"
ts=2026-05-14T08:21:17.918473419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39892"
level=error ts=2026-05-14T08:21:18.164478398Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:21:18.166209822Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:21:18.167740717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:21:18.167759966Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:21:18.81793312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:21:23.42653778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47788"
level=debug ts=2026-05-14T08:21:31.374550784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:21:47.921861617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39906"
ts=2026-05-14T08:21:48.821018155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:22:02.434482023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:38098"
ts=2026-05-14T08:22:17.925007239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46694"
ts=2026-05-14T08:22:18.823442769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:22:23.088336182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:22:23.090118592Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:22:23.431009635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36072"
ts=2026-05-14T08:22:32.436081903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:42562"
ts=2026-05-14T08:22:48.825205999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:23:02.437145477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:59478"
ts=2026-05-14T08:23:17.934788811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56054"
ts=2026-05-14T08:23:18.827806196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:23:19.999026568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:23:23.435307177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47872"
ts=2026-05-14T08:23:32.438842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:50232"
ts=2026-05-14T08:23:48.830721003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:23:53.437392645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59714"
level=debug ts=2026-05-14T08:24:15.399344144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:24:18.833053665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:24:48.835142541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:24:53.441258569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56028"
level=debug ts=2026-05-14T08:24:59.272777999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:25:18.837221169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:25:44.256897966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:25:47.948902481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51600"
ts=2026-05-14T08:25:48.83949984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:26:18.84267372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:26:23.935701178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:26:23.937532115Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:26:48.845953693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:26:53.452172058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34214"
level=debug ts=2026-05-14T08:27:10.530044932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:27:18.84847679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:27:23.453842058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59046"
ts=2026-05-14T08:27:47.959762606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54636"
ts=2026-05-14T08:27:48.850912207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:27:53.45591347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38492"
level=debug ts=2026-05-14T08:27:56.517336009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:28:02.459463772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33118"
ts=2026-05-14T08:28:17.96282975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37464"
ts=2026-05-14T08:28:18.853058159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:28:32.460989758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:56154"
level=debug ts=2026-05-14T08:28:34.154096522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:28:48.854525541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:28:53.460491583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43594"
ts=2026-05-14T08:29:17.969115809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52398"
level=debug ts=2026-05-14T08:29:18.719360775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:29:18.856342276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:29:47.972038966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41590"
ts=2026-05-14T08:29:48.858966952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:30:02.466941224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:41038"
level=debug ts=2026-05-14T08:30:14.408746069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:30:14.410387006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:30:17.975591436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37094"
ts=2026-05-14T08:30:18.86053024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:30:47.97884988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41596"
ts=2026-05-14T08:30:48.862697582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:31:12.03299113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:31:18.162789064Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:31:18.16444027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:31:18.165998194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:31:18.166026941Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:31:18.865389851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:31:23.475322501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42976"
ts=2026-05-14T08:31:48.867949197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:31:59.72263719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:32:02.477286674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:54064"
ts=2026-05-14T08:32:18.869840923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:32:48.87172418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:32:50.984831492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:33:02.481758514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:47486"
ts=2026-05-14T08:33:18.873253099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:33:32.483210188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:53990"
ts=2026-05-14T08:33:48.876021036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:33:51.405585821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:34:18.000937595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58264"
ts=2026-05-14T08:34:18.878109444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:34:23.491637272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42394"
ts=2026-05-14T08:34:48.003717971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51364"
ts=2026-05-14T08:34:48.880871768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:34:52.109562051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:34:52.111252927Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:35:18.883650655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:35:23.496366501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36826"
level=debug ts=2026-05-14T08:35:43.357280148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:35:48.886573582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:35:53.498321485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55634"
ts=2026-05-14T08:36:18.889837539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:36:26.164042714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:36:32.497954371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45114"
ts=2026-05-14T08:36:48.017181121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56998"
ts=2026-05-14T08:36:48.892869907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:37:02.498962509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:33640"
level=debug ts=2026-05-14T08:37:16.306512189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:37:18.019657534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50432"
ts=2026-05-14T08:37:18.895311161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:37:23.506139188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42100"
ts=2026-05-14T08:37:48.022025352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38272"
ts=2026-05-14T08:37:48.897942432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:38:08.32572731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:38:18.90034049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:38:32.504471376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39846"
ts=2026-05-14T08:38:48.028341443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60986"
ts=2026-05-14T08:38:48.90177041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:38:53.513192069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53356"
level=debug ts=2026-05-14T08:38:58.440936461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:38:58.442641354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:39:02.505931324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:34596"
ts=2026-05-14T08:39:18.904699949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:39:38.33385599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:39:48.906345337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:39:53.519220139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49458"
ts=2026-05-14T08:40:02.509619168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:36284"
ts=2026-05-14T08:40:18.037879279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33174"
ts=2026-05-14T08:40:18.908756033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:40:31.150806149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:40:48.910797116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:40:53.52350257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52888"
ts=2026-05-14T08:41:02.513784694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:39852"
level=error ts=2026-05-14T08:41:18.164576942Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:41:18.166329759Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:41:18.168302491Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-14T08:41:18.168326183Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:41:18.912783305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:41:23.52540145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46858"
level=debug ts=2026-05-14T08:41:27.607569998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:41:48.914342415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:41:53.527308669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38666"
level=debug ts=2026-05-14T08:42:05.007318611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:42:18.916091538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:42:48.918055239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:42:52.31652273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:42:52.318046911Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:43:18.056598957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60826"
ts=2026-05-14T08:43:18.920057681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
ts=2026-05-14T08:43:23.534050709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60496"
level=debug ts=2026-05-14T08:43:27.195778884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:43:48.921808031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:44:02.529957691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:58870"
level=debug ts=2026-05-14T08:44:16.694333459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:44:18.924543091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
ts=2026-05-14T08:44:23.539105242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46766"
ts=2026-05-14T08:44:32.53094326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.85:45198"
ts=2026-05-14T08:44:48.066493737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53972"
ts=2026-05-14T08:44:48.927477138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dn9ps-b2c83a3a 10.129.0.69:7946"
level=debug ts=2026-05-14T08:45:07.67614652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:45:18.929566361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-f2zhb-517b3857 10.130.0.69:7946"
level=debug ts=2026-05-14T08:45:46.715914822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:45:48.93071447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:45:53.545986444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41140"
ts=2026-05-14T08:46:18.074907706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43134"
ts=2026-05-14T08:46:18.931818127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:46:23.873989323Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-14T08:46:23.875665072Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-14T08:46:48.078386302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56044"
ts=2026-05-14T08:46:48.933288632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
ts=2026-05-14T08:46:53.551252158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46724"
ts=2026-05-14T08:47:18.934654293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:47:25.005552425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-14T08:47:48.084976066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52620"
ts=2026-05-14T08:47:48.93591576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00741a9e 10.128.0.85:7946"
level=debug ts=2026-05-14T08:48:02.023680609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"