level=info ts=2026-05-01T22:45:57.431314563Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-01T22:45:57.431338892Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-01T22:45:57.432864233Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-01T22:45:57.433171672Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T22:45:57.43324165Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.86 inf=eth0
level=debug ts=2026-05-01T22:45:57.433610869Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-01T22:45:57.433669374Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.86 inf=eth0
level=info ts=2026-05-01T22:45:57.4337417Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-33c59d6d
ts=2026-05-01T22:45:57.433823446Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-01T22:45:57.433836165Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.86 advertisePort=7946
level=debug ts=2026-05-01T22:45:57.433852361Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.86 advertisePort=7946
level=info ts=2026-05-01T22:45:57.434002447Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-01T22:45:57.440754635Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T22:45:57.440774332Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-01T22:45:57.440788698Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.788996ms
level=error ts=2026-05-01T22:45:57.440801167Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-01T22:45:57.440812269Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-01T22:45:57.446022904Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:45:57.446043269Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T22:45:57.449884798Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-01T22:45:57.449954525Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-01T22:45:57.449956829Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-01T22:45:57.449977018Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-01T22:45:57.450084836Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-01T22:45:57.450104661Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-01T22:45:57.450120481Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-01T22:45:57.450140581Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-01T22:45:57.450491268Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-01T22:45:57.450501725Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-01T22:45:57.450547738Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-01T22:45:57.450589953Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-01T22:45:57.450613602Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-01T22:45:57.450746343Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-01T22:45:57.585854376Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-01T22:45:57.58592919Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-01T22:45:58.586759408Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-01T22:45:58.71318215Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-01T22:45:58.713254389Z caller=loki.go:615 msg="Loki started" startup_time=1.349213912s
ts=2026-05-01T22:45:58.916451421Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:45:58.916475013Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:46:01.620307884Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:46:01.620332538Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-01T22:46:03.71357594Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-01T22:46:03.713646996Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-01T22:46:05.694019142Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-01T22:46:05.694042111Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-01T22:46:16.098063875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:46:18.739794865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
level=info ts=2026-05-01T22:46:18.741047906Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.300229395s
ts=2026-05-01T22:46:22.378960347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36972"
level=debug ts=2026-05-01T22:46:34.109826181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:46:34.966395649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:46:56.537328591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:47:04.968218736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T22:47:24.385672939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44506"
level=debug ts=2026-05-01T22:47:25.435688303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:47:34.969621165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:48:00.494051107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T22:48:00.495708005Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T22:48:04.970937509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:48:20.765041033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42764"
ts=2026-05-01T22:48:34.97320393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:48:49.170320435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:49:04.975454205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T22:49:34.977661722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T22:49:50.770453045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58938"
ts=2026-05-01T22:49:54.396432832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50368"
level=debug ts=2026-05-01T22:49:56.282028295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:50:03.221717596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56304"
ts=2026-05-01T22:50:04.97956448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:50:33.223305053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44954"
ts=2026-05-01T22:50:34.981133996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T22:50:54.400006238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57694"
level=debug ts=2026-05-01T22:50:57.268858026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:51:03.224984099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60192"
ts=2026-05-01T22:51:04.982492497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:51:24.401638514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60816"
ts=2026-05-01T22:51:33.226326168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41184"
ts=2026-05-01T22:51:34.983840463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T22:52:04.985650997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:52:17.303774914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-01T22:52:20.779841725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45186"
ts=2026-05-01T22:52:34.987236732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T22:52:50.781664843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58826"
ts=2026-05-01T22:53:03.2296647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57382"
ts=2026-05-01T22:53:04.989380185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:53:07.63957699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-01T22:53:07.641099777Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T22:53:20.782642215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57192"
ts=2026-05-01T22:53:24.410477584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60946"
ts=2026-05-01T22:53:34.990969227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:53:54.412080134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35892"
level=debug ts=2026-05-01T22:54:01.987299196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:54:04.992053754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:54:20.78528928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57652"
ts=2026-05-01T22:54:33.233137245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51150"
ts=2026-05-01T22:54:34.993541632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:54:41.805247665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:55:04.995265814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:55:26.068368563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:55:34.996694071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:55:50.78981783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34654"
ts=2026-05-01T22:56:03.237029628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57724"
level=info ts=2026-05-01T22:56:03.71456891Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-01T22:56:03.718009027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T22:56:03.719804736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T22:56:03.72162506Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T22:56:03.721650396Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-01T22:56:03.72166276Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-01T22:56:04.998329807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T22:56:13.437587574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:56:35.000128016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:56:50.025695192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T22:56:50.027424228Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T22:57:03.238847015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47028"
ts=2026-05-01T22:57:05.001129896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T22:57:35.002652491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:57:46.890333693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:58:05.00434841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T22:58:34.046474142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:58:35.005170812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T22:59:05.0068088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T22:59:11.303419612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:59:20.801702224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52834"
ts=2026-05-01T22:59:35.00903765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T22:59:47.447232285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T22:59:50.802690455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58232"
ts=2026-05-01T23:00:03.247853781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46892"
ts=2026-05-01T23:00:05.010245104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:00:20.804163013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:44292"
ts=2026-05-01T23:00:24.437112955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54084"
ts=2026-05-01T23:00:35.011971888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:00:40.745607406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:00:40.747204369Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:01:03.250994511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59480"
ts=2026-05-01T23:01:05.013217036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:01:21.446821464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:01:24.44117268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50470"
ts=2026-05-01T23:01:33.252199138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43970"
ts=2026-05-01T23:01:35.014524161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:01:50.808220941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51668"
ts=2026-05-01T23:02:05.015804129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:02:09.779447706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:02:20.80924387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49138"
ts=2026-05-01T23:02:33.255136908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56888"
ts=2026-05-01T23:02:35.017503856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:03:05.019383415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:03:08.316133082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:03:20.812834312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49536"
ts=2026-05-01T23:03:35.021009728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:03:50.752199848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:04:03.260995738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:54068"
ts=2026-05-01T23:04:05.022667234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:04:20.816190162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56230"
ts=2026-05-01T23:04:35.024502616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:04:44.93241628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:04:44.934903142Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:05:05.02622485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:05:35.027690371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:05:44.027138974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:06:03.728639079Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:06:03.731318228Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:06:03.733810483Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:06:03.733868907Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:06:05.029738128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:06:26.663783433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:06:35.032083096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:07:05.033679611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:07:17.385304759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:07:20.828413243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51116"
ts=2026-05-01T23:07:35.035622438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:08:05.037455072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:08:11.951992245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:08:20.832710744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51488"
ts=2026-05-01T23:08:35.038952956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:08:50.83472139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34052"
level=debug ts=2026-05-01T23:08:58.325030789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:08:58.327286061Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:09:05.040831991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:09:24.477082537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48264"
ts=2026-05-01T23:09:35.042384705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:09:54.479417867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35714"
level=debug ts=2026-05-01T23:09:55.91792718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:10:05.043668504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:10:35.044558166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:10:38.97126137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:11:03.286958691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36932"
ts=2026-05-01T23:11:05.045382704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:11:24.485468961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58624"
level=debug ts=2026-05-01T23:11:32.65991053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:11:35.04629777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:12:05.047854263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:12:08.636704794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:12:20.845528944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58312"
ts=2026-05-01T23:12:33.292274844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59330"
ts=2026-05-01T23:12:35.050360171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:12:50.846547927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49620"
level=debug ts=2026-05-01T23:12:55.70814904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:12:55.710092075Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:13:05.052724578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:13:24.492968621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39050"
ts=2026-05-01T23:13:35.05546347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:13:46.433895665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:14:05.056967997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:14:24.49684212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48766"
ts=2026-05-01T23:14:35.058645373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:14:42.232159479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:15:05.060210396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:15:20.855622685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55716"
level=debug ts=2026-05-01T23:15:29.598996149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:15:33.30523416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33622"
ts=2026-05-01T23:15:35.061822492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=error ts=2026-05-01T23:16:03.726400103Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:16:03.728764987Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:16:03.730729272Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:16:03.730752351Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:16:05.06308502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:16:13.325545425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:16:20.859760175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43532"
ts=2026-05-01T23:16:24.507180008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52676"
ts=2026-05-01T23:16:35.064309989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:16:54.513087084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33286"
level=debug ts=2026-05-01T23:16:55.898189886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:16:55.900782472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:17:03.310149559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39420"
ts=2026-05-01T23:17:05.065340992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:17:20.864004396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43104"
level=debug ts=2026-05-01T23:17:30.69774055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:17:35.066436143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:17:50.865893614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34628"
ts=2026-05-01T23:18:03.312615351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50932"
ts=2026-05-01T23:18:05.067835888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:18:05.733178949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:18:20.866997638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50536"
ts=2026-05-01T23:18:33.313716612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46596"
ts=2026-05-01T23:18:35.069282407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:18:48.808258748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:18:50.86854674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:57406"
ts=2026-05-01T23:19:05.070628408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:19:20.870055499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56532"
level=debug ts=2026-05-01T23:19:24.223354895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:19:24.529079075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56284"
ts=2026-05-01T23:19:35.072401396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:19:50.871463326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38864"
ts=2026-05-01T23:19:54.533569027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45220"
level=debug ts=2026-05-01T23:20:01.236326424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:20:01.238220414Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:20:05.073610833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:20:35.075417931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:20:36.554617678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:21:05.076611253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:21:14.517933671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:21:24.539897717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59266"
ts=2026-05-01T23:21:33.32341796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60704"
ts=2026-05-01T23:21:35.07859849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:21:59.712890758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:22:05.080752694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:22:20.879279792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37206"
ts=2026-05-01T23:22:35.082336048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:22:51.904618325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:23:05.08377822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:23:20.882173912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:45730"
ts=2026-05-01T23:23:35.085920734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:23:41.174015751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:23:41.175852349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:23:54.550372696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50266"
ts=2026-05-01T23:24:03.332141301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48264"
ts=2026-05-01T23:24:05.087624815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:24:20.886682854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48956"
ts=2026-05-01T23:24:24.552648469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49786"
ts=2026-05-01T23:24:33.333570988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60748"
ts=2026-05-01T23:24:35.089393595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:24:36.021926949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:25:05.090317523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:25:33.032542213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:25:33.336742882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60502"
ts=2026-05-01T23:25:35.091977057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:26:03.33788745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39888"
level=error ts=2026-05-01T23:26:03.728707939Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:26:03.730493597Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:26:03.732179724Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:26:03.732209786Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:26:05.093088089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:26:13.487123538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:26:35.09454336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:27:02.064960637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:27:03.341413635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:38520"
ts=2026-05-01T23:27:05.096107624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:27:20.89633324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47380"
ts=2026-05-01T23:27:24.565184685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42114"
ts=2026-05-01T23:27:35.097240283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:27:49.176423418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:27:49.178781375Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:27:54.566677637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58196"
ts=2026-05-01T23:28:05.098889636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:28:20.899829946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:38882"
level=debug ts=2026-05-01T23:28:28.154564049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:28:33.345625846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57110"
ts=2026-05-01T23:28:35.101115529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:28:54.570492102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40866"
ts=2026-05-01T23:29:05.102031984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:29:17.617737853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:29:35.103730908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:29:55.810647987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:30:05.104903469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:30:20.907742476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39584"
ts=2026-05-01T23:30:24.57676965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58788"
ts=2026-05-01T23:30:35.106507513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:30:44.755754187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:31:05.108342217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:31:28.528564554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:31:28.530224876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:31:35.109260206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:32:03.356611646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37422"
ts=2026-05-01T23:32:05.110281111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:32:24.586098284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57544"
level=debug ts=2026-05-01T23:32:30.11362757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:32:35.113399645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:32:50.918740729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39802"
ts=2026-05-01T23:33:05.115375461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:33:13.206835779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:33:24.591045056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33558"
ts=2026-05-01T23:33:33.360413344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36728"
ts=2026-05-01T23:33:35.116990074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:33:49.336541284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:34:05.118475546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:34:24.595356691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33276"
level=debug ts=2026-05-01T23:34:27.328012032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:34:33.364052274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48278"
ts=2026-05-01T23:34:35.120086545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:35:03.365038386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50438"
ts=2026-05-01T23:35:05.121531021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:35:10.121597602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:35:10.123074337Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:35:24.599201703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46878"
ts=2026-05-01T23:35:33.366186596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:38524"
ts=2026-05-01T23:35:35.123226125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=error ts=2026-05-01T23:36:03.726312583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:36:03.728244458Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:36:03.730042224Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:36:03.730066819Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:36:05.12537901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:36:09.866475607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:36:35.127327048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:36:54.604541187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56904"
level=debug ts=2026-05-01T23:36:59.135861909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:37:05.129066462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:37:24.606870406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60976"
ts=2026-05-01T23:37:35.13059958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:37:46.746748379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:38:05.132049637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:38:22.205577559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:38:35.133138154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:39:05.135018271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:39:11.848387469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:39:11.85068875Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:39:35.136192881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:39:54.622003413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53024"
level=debug ts=2026-05-01T23:40:02.815658091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:40:03.379031572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33718"
ts=2026-05-01T23:40:05.137439096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:40:20.945063132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37918"
ts=2026-05-01T23:40:24.624131412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42784"
ts=2026-05-01T23:40:35.139548044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:40:54.626626849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:32862"
level=debug ts=2026-05-01T23:40:58.344534526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:41:03.381699435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:32966"
ts=2026-05-01T23:41:05.140489136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:41:35.145483095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:41:45.763852377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:41:54.631042574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58640"
ts=2026-05-01T23:42:05.146415418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:42:32.35077653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:42:35.147936675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:42:50.954879401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:41632"
ts=2026-05-01T23:42:54.635048366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56958"
ts=2026-05-01T23:43:05.14914386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:43:07.438284354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:43:07.439817158Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:43:35.150643775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:43:47.520868545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:43:54.639279038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38414"
ts=2026-05-01T23:44:03.391175371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52616"
ts=2026-05-01T23:44:05.152272339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:44:20.959210102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39692"
ts=2026-05-01T23:44:24.641368008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55382"
ts=2026-05-01T23:44:35.154496512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:44:42.208887967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:45:05.156805569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:45:20.962840306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:55396"
level=debug ts=2026-05-01T23:45:24.092068583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:45:35.158789245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:45:50.964035827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53622"
ts=2026-05-01T23:45:54.647303172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38124"
level=debug ts=2026-05-01T23:46:03.343695629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:46:03.398072792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53370"
level=error ts=2026-05-01T23:46:03.724788641Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:46:03.726694911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:46:03.728380557Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:46:03.72840099Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:46:05.161022235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:46:35.163134295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:46:45.338261289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:46:45.34008754Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:47:05.165025333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:47:26.147517943Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:47:35.166558957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:47:50.970470146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52934"
ts=2026-05-01T23:48:03.404628797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34646"
ts=2026-05-01T23:48:05.168430661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:48:20.039956608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:48:24.658709551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:40558"
ts=2026-05-01T23:48:35.170539305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:48:50.974347341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:47808"
ts=2026-05-01T23:48:54.660569989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41644"
ts=2026-05-01T23:49:05.17258954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:49:14.287367747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:49:20.976013368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48648"
ts=2026-05-01T23:49:24.662265915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57992"
ts=2026-05-01T23:49:33.409088603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44648"
ts=2026-05-01T23:49:35.175005753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:49:50.977095033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:39762"
ts=2026-05-01T23:49:54.66407588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58406"
ts=2026-05-01T23:50:03.410903493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46246"
ts=2026-05-01T23:50:05.176916746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-01T23:50:06.136967466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:50:33.412614779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49056"
ts=2026-05-01T23:50:35.178006908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:51:02.784262366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:51:02.78604124Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:51:05.17969792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:51:35.182157187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:51:50.983451301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:40694"
ts=2026-05-01T23:51:54.672274047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38202"
level=debug ts=2026-05-01T23:51:56.38019032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:52:03.418280914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:45544"
ts=2026-05-01T23:52:05.183706501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:52:33.419825526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56176"
level=debug ts=2026-05-01T23:52:35.179212876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:52:35.185376745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:53:03.421675748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35714"
ts=2026-05-01T23:53:05.186660127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:53:24.678332493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:32932"
level=debug ts=2026-05-01T23:53:26.29024941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:53:35.188812273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:53:54.680287089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35884"
ts=2026-05-01T23:54:05.190441298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:54:18.031822522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:54:20.991441552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:54354"
ts=2026-05-01T23:54:33.42617993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56178"
ts=2026-05-01T23:54:35.191661168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:55:05.193282369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:55:18.759473882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:55:18.761030731Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:55:33.430063283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57744"
ts=2026-05-01T23:55:35.194541018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:55:50.996337525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56952"
level=error ts=2026-05-01T23:56:03.725670312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:56:03.727537835Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:56:03.729065818Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-01T23:56:03.729088816Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:56:05.196101652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:56:08.995761753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:56:20.998086691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37012"
ts=2026-05-01T23:56:24.689748866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54326"
ts=2026-05-01T23:56:35.197834841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-01T23:56:51.86244945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:56:54.692127716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50322"
ts=2026-05-01T23:57:05.199426344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:57:21.000439437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:51878"
level=debug ts=2026-05-01T23:57:29.202688582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:57:35.200690181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-01T23:57:51.001228365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:59872"
level=debug ts=2026-05-01T23:58:04.868434742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-01T23:58:05.202767932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-01T23:58:35.204351545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:58:48.805284722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-01T23:58:48.806833408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-01T23:59:05.205510899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-01T23:59:21.006324745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49064"
ts=2026-05-01T23:59:24.702093633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49234"
ts=2026-05-01T23:59:35.207063071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-01T23:59:46.258010803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:00:03.445842734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33602"
ts=2026-05-02T00:00:05.207976981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:00:24.17439435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:00:24.706239783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55520"
ts=2026-05-02T00:00:35.209912687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:00:51.010923213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:43772"
ts=2026-05-02T00:00:54.708101628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47884"
ts=2026-05-02T00:01:05.212094479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:01:13.313572289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:01:24.710252185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45096"
ts=2026-05-02T00:01:33.448658099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44020"
ts=2026-05-02T00:01:35.212995769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:01:49.706626432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:02:05.214500874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:02:21.015342943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:58704"
ts=2026-05-02T00:02:24.714887306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56310"
ts=2026-05-02T00:02:35.216178974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:02:39.44028926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:02:39.44176688Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:02:54.717102545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58792"
ts=2026-05-02T00:03:05.217674753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:03:14.82704223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:03:33.455889367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48674"
ts=2026-05-02T00:03:35.219101397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:03:51.020294798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:42614"
level=debug ts=2026-05-02T00:04:00.080222148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:04:05.219919383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:04:33.458103035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59668"
ts=2026-05-02T00:04:35.222056365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:04:51.022413134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35280"
level=debug ts=2026-05-02T00:04:53.586343657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:05:03.459091029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59336"
ts=2026-05-02T00:05:05.223332611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:05:35.225150592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:05:41.933147059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:06:03.461790355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52594"
level=error ts=2026-05-02T00:06:03.72562628Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:06:03.727400997Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:06:03.729039432Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:06:03.729063256Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:06:05.22723667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:06:25.858492545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:06:25.86027026Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:06:35.22886671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:07:05.230638114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:07:20.585103024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:07:21.029866891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:34280"
ts=2026-05-02T00:07:33.466221997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48942"
ts=2026-05-02T00:07:35.231724774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:07:51.030954019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:60346"
ts=2026-05-02T00:08:03.468078266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59836"
ts=2026-05-02T00:08:05.233136173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:08:21.031927542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35008"
level=debug ts=2026-05-02T00:08:22.20246367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:08:33.469485387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41814"
ts=2026-05-02T00:08:35.235174205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:09:03.470530199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51400"
ts=2026-05-02T00:09:05.236832314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:09:09.533462672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:09:33.471872527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41420"
ts=2026-05-02T00:09:35.239065094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:10:05.240097414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:10:09.941416884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:10:21.037719505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46352"
ts=2026-05-02T00:10:35.241316778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:10:51.03897163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:52020"
ts=2026-05-02T00:11:03.476046668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37876"
level=debug ts=2026-05-02T00:11:03.559561455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:11:03.561027465Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:11:05.242233953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:11:21.040640033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:56374"
ts=2026-05-02T00:11:35.243582912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:11:51.042204609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:46606"
ts=2026-05-02T00:11:54.752901231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34548"
level=debug ts=2026-05-02T00:11:59.624414041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:12:05.24476438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:12:35.24631347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:12:51.244636593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:12:54.757289425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49390"
ts=2026-05-02T00:13:03.482891104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35510"
ts=2026-05-02T00:13:05.247903343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:13:24.759698888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38336"
level=debug ts=2026-05-02T00:13:26.745752985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:13:33.483801915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33564"
ts=2026-05-02T00:13:35.249208196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:13:54.762041038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33078"
ts=2026-05-02T00:14:03.485205855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36606"
ts=2026-05-02T00:14:05.251251245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:14:16.137547538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:14:21.050235994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49982"
ts=2026-05-02T00:14:24.764171213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50058"
ts=2026-05-02T00:14:33.486644747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50786"
ts=2026-05-02T00:14:35.253189193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:15:03.238213312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:15:03.239991619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:15:03.48787542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35534"
ts=2026-05-02T00:15:05.254691979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:15:35.256169236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:15:47.821822812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:16:03.725655252Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:16:03.727688326Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:16:03.729518551Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:16:03.729545674Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:16:05.258431469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:16:24.77277637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:60136"
ts=2026-05-02T00:16:33.493015127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53490"
ts=2026-05-02T00:16:35.260111961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:16:40.367549087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:17:03.494155622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55126"
ts=2026-05-02T00:17:05.261752636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:17:33.495000417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48624"
ts=2026-05-02T00:17:35.262687296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:17:38.736978636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:17:51.062845224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:48288"
ts=2026-05-02T00:17:54.778687415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:46438"
ts=2026-05-02T00:18:05.263609957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:18:29.651449728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:18:35.264575438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:19:05.265598511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:19:06.76358737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:19:06.765355027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:19:35.267185928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:20:05.268996696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:20:07.672975776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:20:24.787593835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53384"
ts=2026-05-02T00:20:35.270264778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:20:47.339106449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:21:05.272020976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:21:21.072299253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:50790"
ts=2026-05-02T00:21:33.512508604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59452"
ts=2026-05-02T00:21:35.272912263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:21:43.612228723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:22:03.513532956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51174"
ts=2026-05-02T00:22:05.274563936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:22:21.075158369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35356"
ts=2026-05-02T00:22:35.275843358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:22:44.073505555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:22:54.797104745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43436"
ts=2026-05-02T00:23:03.516234062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39152"
ts=2026-05-02T00:23:05.277980789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:23:21.078505404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:33798"
ts=2026-05-02T00:23:35.280354539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:23:41.588131386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:23:41.58953558Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:23:51.080166254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:49542"
ts=2026-05-02T00:24:05.282350292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:24:24.803831894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58906"
ts=2026-05-02T00:24:33.520683507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60802"
ts=2026-05-02T00:24:35.283554664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:24:43.531210422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:25:03.522615935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46522"
ts=2026-05-02T00:25:05.284998618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
level=debug ts=2026-05-02T00:25:20.391911792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:25:35.287472312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:25:51.086751524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:37576"
level=debug ts=2026-05-02T00:26:01.892380534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:26:03.527172426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39472"
level=error ts=2026-05-02T00:26:03.724426971Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:26:03.726174548Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:26:03.728034596Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-02T00:26:03.728061445Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:26:05.2894266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"
ts=2026-05-02T00:26:21.088447994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:36224"
ts=2026-05-02T00:26:35.291014988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:26:42.803346792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:26:51.089834084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:53098"
ts=2026-05-02T00:27:05.292803256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
level=debug ts=2026-05-02T00:27:34.904877854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-02T00:27:34.906383959Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-02T00:27:35.294737661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:27:54.819694913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48886"
ts=2026-05-02T00:28:03.533558629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53798"
ts=2026-05-02T00:28:05.296021302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
level=debug ts=2026-05-02T00:28:19.390315011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:28:24.82226468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57700"
ts=2026-05-02T00:28:33.534370152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:53208"
ts=2026-05-02T00:28:35.297625799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:28:54.824130894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54568"
level=debug ts=2026-05-02T00:29:03.32075504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:29:03.535214357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:44920"
ts=2026-05-02T00:29:05.299176378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lx78r-82ff3e96 10.129.0.76:7946"
ts=2026-05-02T00:29:35.300852227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-b8rcw-1efd5fad 10.128.0.84:7946"
ts=2026-05-02T00:29:51.099521908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.84:35978"
ts=2026-05-02T00:29:54.828784171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47826"
level=debug ts=2026-05-02T00:29:59.774103609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-02T00:30:03.538137285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57980"
ts=2026-05-02T00:30:05.302089093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8fc2a50b 10.128.0.87:7946"