level=info ts=2026-04-28T09:47:36.634488204Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-28T09:47:36.634521814Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-28T09:47:36.636297268Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T09:47:36.636578012Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:47:36.636645438Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=debug ts=2026-04-28T09:47:36.637027955Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:47:36.637081357Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=info ts=2026-04-28T09:47:36.637150392Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-0e9723a5
ts=2026-04-28T09:47:36.637210681Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T09:47:36.637241287Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=debug ts=2026-04-28T09:47:36.637261196Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=info ts=2026-04-28T09:47:36.637356931Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T09:47:36.644660735Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T09:47:36.644681262Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T09:47:36.644694185Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.339074ms
level=error ts=2026-04-28T09:47:36.644703955Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-28T09:47:36.644714888Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-28T09:47:36.64751719Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:47:36.647531834Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T09:47:36.652189583Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-28T09:47:36.652190345Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-28T09:47:36.652214785Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-28T09:47:36.652254555Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-28T09:47:36.652278014Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T09:47:36.652266036Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-28T09:47:36.652360162Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-28T09:47:36.652378876Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-28T09:47:36.652490298Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-28T09:47:36.65249642Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-28T09:47:36.652557096Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-28T09:47:36.652552657Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-28T09:47:36.652575315Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T09:47:36.652747798Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-28T09:47:36.652757132Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-28T09:47:36.652781311Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-28T09:47:37.653399924Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:47:37.755638745Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:47:37.755689358Z caller=loki.go:599 msg="Loki started" startup_time=1.180942708s
ts=2026-04-28T09:47:38.169237786Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:47:38.16926116Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:47:40.722172354Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:47:40.722198594Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T09:47:42.756709997Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-28T09:47:42.756760666Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-28T09:47:48.01046805Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:47:48.01049049Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T09:47:54.465590443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:47:57.892476136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.66:7946"
ts=2026-04-28T09:47:57.895634312Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.67:7946: dial tcp 10.128.0.67:7946: connect: connection refused"
level=info ts=2026-04-28T09:47:57.895650914Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.250929309s
ts=2026-04-28T09:48:09.268569832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T09:48:13.754530859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:48:18.763182911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34624"
ts=2026-04-28T09:48:30.107206668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43278"
level=debug ts=2026-04-28T09:48:32.979858347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:48:39.271542205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T09:48:48.766810126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34572"
level=debug ts=2026-04-28T09:49:00.602001653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:49:09.274103715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T09:49:18.769930639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36266"
ts=2026-04-28T09:49:29.907947233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49750"
level=debug ts=2026-04-28T09:49:32.319566246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T09:49:32.321208595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:49:39.276289419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:49:59.911624561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50664"
ts=2026-04-28T09:50:09.279169778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:50:18.775563542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42006"
level=debug ts=2026-04-28T09:50:23.877699304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:50:30.118570707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47620"
ts=2026-04-28T09:50:39.282821541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T09:50:48.779035547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:43464"
ts=2026-04-28T09:51:09.28504322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T09:51:27.055445076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:51:39.286510247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T09:51:48.785096269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38066"
ts=2026-04-28T09:51:59.923160969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37066"
ts=2026-04-28T09:52:09.28755611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T09:52:18.787943397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46544"
level=debug ts=2026-04-28T09:52:30.388255669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:52:39.290092996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:52:59.929274692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38958"
ts=2026-04-28T09:53:00.133202813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37966"
ts=2026-04-28T09:53:09.292011279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T09:53:29.932297304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39606"
ts=2026-04-28T09:53:39.29479239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T09:53:42.478660868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:54:00.137210039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43122"
ts=2026-04-28T09:54:09.297839191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:54:30.138025885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59962"
level=debug ts=2026-04-28T09:54:33.86119448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T09:54:33.862865751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:54:39.300940707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T09:55:09.302589356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T09:55:18.805463293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44056"
level=debug ts=2026-04-28T09:55:29.512944276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:55:39.30464234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:56:09.3065465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T09:56:12.880419262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:56:39.30863666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T09:56:57.855364488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:57:00.15266417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35558"
ts=2026-04-28T09:57:09.312053422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:57:39.315094751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=info ts=2026-04-28T09:57:42.757532004Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-28T09:57:42.761063514Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:57:42.763082814Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:57:42.764936206Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:57:42.764964016Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T09:57:42.764994476Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-28T09:57:48.820237249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54084"
level=debug ts=2026-04-28T09:57:55.002574106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:57:59.954283599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38860"
ts=2026-04-28T09:58:09.318465408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:58:18.823380154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39466"
level=debug ts=2026-04-28T09:58:37.664153608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T09:58:37.66569785Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:58:39.32217642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T09:58:48.826675187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42632"
ts=2026-04-28T09:58:59.959858732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50992"
ts=2026-04-28T09:59:09.325134649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T09:59:17.857147955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:59:18.830108409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35256"
ts=2026-04-28T09:59:39.32816441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:00:05.051486861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:00:09.330072938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:00:39.258902199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:00:39.332180569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:00:48.838993831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49364"
ts=2026-04-28T10:00:59.969418893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34096"
ts=2026-04-28T10:01:00.174966034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33396"
ts=2026-04-28T10:01:09.334250638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:01:21.652345603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:01:29.972827754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33426"
ts=2026-04-28T10:01:39.336219422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:02:00.180016278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43976"
level=debug ts=2026-04-28T10:02:05.206175204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:02:05.208268326Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:02:09.33952498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:02:18.847954809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:50052"
ts=2026-04-28T10:02:39.342741405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:02:48.851423859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48904"
level=debug ts=2026-04-28T10:02:57.507615468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:03:09.345634123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:03:18.855164861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49126"
ts=2026-04-28T10:03:29.984126221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44050"
ts=2026-04-28T10:03:39.348965826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:03:45.303378392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:03:59.987693606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52336"
ts=2026-04-28T10:04:00.190478391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50484"
ts=2026-04-28T10:04:09.350918813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:04:29.990594382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36964"
level=debug ts=2026-04-28T10:04:34.898907665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:04:39.35232451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:04:59.993174627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34038"
ts=2026-04-28T10:05:09.354009403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:05:18.869245503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58664"
ts=2026-04-28T10:05:29.996817704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50744"
level=debug ts=2026-04-28T10:05:34.081994859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:05:39.355850055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:06:00.001651621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50734"
ts=2026-04-28T10:06:00.201557998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38528"
ts=2026-04-28T10:06:09.359006225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:06:18.874198295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45942"
level=debug ts=2026-04-28T10:06:24.085320236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:06:24.087175909Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:06:30.004640064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49410"
ts=2026-04-28T10:06:39.360933667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:06:48.877037965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55316"
level=debug ts=2026-04-28T10:07:00.295799355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:07:09.361888642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:07:36.500452027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:07:39.36443678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=error ts=2026-04-28T10:07:42.768457058Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:07:42.770952208Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:07:42.773359499Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:07:42.773387927Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:08:00.214576588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53250"
ts=2026-04-28T10:08:09.374110421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:08:18.891202119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:41766"
ts=2026-04-28T10:08:30.017234594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49576"
ts=2026-04-28T10:08:30.216562674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37436"
level=debug ts=2026-04-28T10:08:33.594060109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:08:39.376236474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:08:48.894520076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55646"
ts=2026-04-28T10:09:00.020756051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:58744"
ts=2026-04-28T10:09:09.37803947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:09:16.510720994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:09:30.220340546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58942"
ts=2026-04-28T10:09:39.380149328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:09:48.899538038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55824"
level=debug ts=2026-04-28T10:09:52.798457656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:09:52.800044813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:10:09.382353026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:10:29.618669021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:10:30.028175562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50308"
ts=2026-04-28T10:10:39.384122923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:11:09.385936728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:11:18.543101316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:11:18.910343753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52926"
ts=2026-04-28T10:11:30.232290139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35796"
ts=2026-04-28T10:11:39.388394666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:12:00.037321808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50336"
ts=2026-04-28T10:12:00.234123752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48398"
level=debug ts=2026-04-28T10:12:03.712454123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:12:09.390432055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:12:30.236326184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33316"
ts=2026-04-28T10:12:39.391915812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:13:02.312926593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:13:09.394333031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:13:18.924626603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:50998"
level=debug ts=2026-04-28T10:13:39.237644629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:13:39.239895064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:13:39.396511426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:14:00.244675923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37092"
ts=2026-04-28T10:14:09.398593055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:14:18.930905078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52064"
ts=2026-04-28T10:14:30.051119395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57456"
ts=2026-04-28T10:14:39.400451162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:14:40.969281453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:15:09.40247133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:15:19.376236593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:15:39.40512892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=debug ts=2026-04-28T10:15:59.932925102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:00.255672068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:56914"
ts=2026-04-28T10:16:09.407080134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:16:30.25640132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60172"
level=debug ts=2026-04-28T10:16:37.557025705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:39.408855948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
ts=2026-04-28T10:17:00.064139001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57578"
ts=2026-04-28T10:17:09.410509835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:17:15.104106071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:17:15.105956275Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:17:18.947281583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42676"
ts=2026-04-28T10:17:30.067122853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37410"
ts=2026-04-28T10:17:39.412670155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-502d3e50 10.130.0.72:7946"
level=error ts=2026-04-28T10:17:42.771839027Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:17:42.77461033Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:17:42.7769507Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:17:42.777003694Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:18:00.070626194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42102"
level=debug ts=2026-04-28T10:18:03.828428148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:18:05.778846518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38536"
ts=2026-04-28T10:18:05.794352592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38538"
ts=2026-04-28T10:18:09.413599273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
ts=2026-04-28T10:18:13.638874397Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-502d3e50 (timeout reached)"
ts=2026-04-28T10:18:16.639136215Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-502d3e50 has failed, no acks received"
ts=2026-04-28T10:18:33.638240406Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-502d3e50 (timeout reached)"
ts=2026-04-28T10:18:36.638300698Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-502d3e50 has failed, no acks received"
ts=2026-04-28T10:18:36.639380629Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-502d3e50 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:18:39.416441029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:18:48.955687372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48276"
level=debug ts=2026-04-28T10:19:04.507410443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:19:09.423155091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:19:39.426955522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:19:44.466691604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:19:48.961455275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:37722"
ts=2026-04-28T10:19:58.475253553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36658"
ts=2026-04-28T10:20:09.430506544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:20:23.629899125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:20:39.432236556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
ts=2026-04-28T10:20:58.480297341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51010"
ts=2026-04-28T10:21:09.433070773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
level=debug ts=2026-04-28T10:21:18.588667249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:21:18.590575336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:21:30.088518737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59492"
ts=2026-04-28T10:21:39.435571294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:21:58.484759913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52968"
ts=2026-04-28T10:22:00.091616878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53498"
ts=2026-04-28T10:22:09.439279458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:22:16.847601355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:22:30.114099298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41074"
ts=2026-04-28T10:22:39.442935445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:22:48.983031986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38270"
level=debug ts=2026-04-28T10:22:55.916102498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:22:58.489581887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38538"
ts=2026-04-28T10:23:09.445606371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
ts=2026-04-28T10:23:28.491087305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59518"
level=debug ts=2026-04-28T10:23:39.257051396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:23:39.446801669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
ts=2026-04-28T10:23:58.4920428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37016"
ts=2026-04-28T10:24:00.123697867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60260"
ts=2026-04-28T10:24:09.447775286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d9b64d4d 10.130.0.72:7946"
ts=2026-04-28T10:24:18.992108819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39672"
level=debug ts=2026-04-28T10:24:31.752354058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:24:39.450191401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:24:58.496235398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42052"
ts=2026-04-28T10:25:09.453454435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:25:30.132569121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55620"
level=debug ts=2026-04-28T10:25:33.154472408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:25:33.156794847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:25:39.456890064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:25:58.501259646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54332"
ts=2026-04-28T10:26:00.135205643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49988"
ts=2026-04-28T10:26:09.460815066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:26:25.681909647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:26:28.503040438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37342"
ts=2026-04-28T10:26:39.463928504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:26:58.504796808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37566"
level=debug ts=2026-04-28T10:27:07.453238808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:27:09.46680944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:27:28.505747926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37724"
ts=2026-04-28T10:27:30.144032848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33302"
ts=2026-04-28T10:27:39.470117255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=error ts=2026-04-28T10:27:42.770362437Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:27:42.772999641Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:27:42.775453081Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:27:42.775490983Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:27:43.638241279Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d9b64d4d (timeout reached)"
ts=2026-04-28T10:27:46.638051937Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d9b64d4d has failed, no acks received"
ts=2026-04-28T10:27:49.013167787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54300"
ts=2026-04-28T10:27:55.573943522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45130"
ts=2026-04-28T10:27:55.586569089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45134"
level=debug ts=2026-04-28T10:27:56.335521651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:28:05.051594999Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-d9b64d4d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:28:09.473213771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:28:19.016966206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45218"
ts=2026-04-28T10:28:30.150041849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53632"
ts=2026-04-28T10:28:39.475779591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
level=debug ts=2026-04-28T10:28:46.55330517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:28:49.020371744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35138"
ts=2026-04-28T10:29:09.478095589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:29:29.440403028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:29:29.44201829Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:29:39.481701319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:30:09.485267486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:30:15.65369462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:30:19.029528457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34668"
ts=2026-04-28T10:30:39.488202341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
ts=2026-04-28T10:30:49.03306469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38972"
ts=2026-04-28T10:31:03.595351564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34542"
ts=2026-04-28T10:31:09.491135069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:31:17.573313656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:31:30.167291013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35470"
ts=2026-04-28T10:31:39.494568912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:31:49.038071751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53142"
ts=2026-04-28T10:32:00.170248835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36470"
ts=2026-04-28T10:32:03.600050944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37794"
ts=2026-04-28T10:32:09.497841821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:32:16.366431549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:32:39.500519917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:32:56.057280102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:33:09.504251816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:33:33.608482893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60214"
level=debug ts=2026-04-28T10:33:33.873666838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:33:33.875356683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:33:39.507258612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:33:49.052696631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36724"
ts=2026-04-28T10:34:09.50908379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
level=debug ts=2026-04-28T10:34:13.160693515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:34:30.183860774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39932"
ts=2026-04-28T10:34:39.511192858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:34:58.911433754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:35:00.186999936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43750"
ts=2026-04-28T10:35:03.615697498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39882"
ts=2026-04-28T10:35:09.512939347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
ts=2026-04-28T10:35:19.063157452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48070"
ts=2026-04-28T10:35:39.514713969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:35:56.626916419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:36:09.517696565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:36:19.07053274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:41256"
ts=2026-04-28T10:36:30.196901079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48588"
level=debug ts=2026-04-28T10:36:30.990460597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:36:33.623013198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45372"
ts=2026-04-28T10:36:39.520680355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:36:49.073259188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55186"
ts=2026-04-28T10:37:09.523389504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:37:29.899573651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:37:29.90210396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:37:30.202687969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51178"
ts=2026-04-28T10:37:39.525300312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
level=error ts=2026-04-28T10:37:42.770322907Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:37:42.772289021Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:37:42.774116711Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:37:42.774142163Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:37:49.078726688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59204"
ts=2026-04-28T10:38:09.526868269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
level=debug ts=2026-04-28T10:38:10.880332554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:38:39.528275229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
ts=2026-04-28T10:39:00.212612271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34526"
ts=2026-04-28T10:39:03.637017193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46616"
level=debug ts=2026-04-28T10:39:07.85196558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:39:09.530009777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
ts=2026-04-28T10:39:39.532222943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:40:01.72865148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:09.535122114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:40:37.03340191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:39.538150343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:41:09.540794357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:41:19.1018457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42078"
ts=2026-04-28T10:41:30.228457893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55168"
level=debug ts=2026-04-28T10:41:34.070578073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:41:34.072245008Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:41:39.543497666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
ts=2026-04-28T10:41:49.105509673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52904"
ts=2026-04-28T10:42:09.544463488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-058e7de7 10.130.0.72:7946"
level=debug ts=2026-04-28T10:42:17.044506595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:42:39.546562491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:43:00.236384994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50324"
level=debug ts=2026-04-28T10:43:02.562902451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:43:09.550351045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:43:19.114907094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46152"
ts=2026-04-28T10:43:33.664673767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33126"
ts=2026-04-28T10:43:39.553487388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:43:51.68850515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:44:04.151004983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52920"
ts=2026-04-28T10:44:04.163167762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52922"
ts=2026-04-28T10:44:09.556417497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:44:28.638824059Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-058e7de7 (timeout reached)"
ts=2026-04-28T10:44:31.639004436Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-058e7de7 has failed, no acks received"
ts=2026-04-28T10:44:34.250568497Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-058e7de7 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:44:35.404614704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38304"
ts=2026-04-28T10:44:39.559910534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:44:49.663764496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:45:09.561925369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:45:19.127373623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:56284"
level=debug ts=2026-04-28T10:45:37.201000271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:45:37.203058719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:45:39.563307017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:46:00.251652175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47922"
ts=2026-04-28T10:46:09.565294361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:46:27.786187273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:46:35.415299216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35364"
ts=2026-04-28T10:46:39.568439546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:46:49.137522861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55176"
ts=2026-04-28T10:47:05.41691658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35294"
ts=2026-04-28T10:47:09.570957483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:47:14.624740961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:47:30.260700956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33658"
ts=2026-04-28T10:47:39.57279438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
level=error ts=2026-04-28T10:47:42.769035808Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:47:42.770903032Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:47:42.772585183Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:47:42.772608207Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T10:47:56.38511529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:48:00.263717002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37120"
ts=2026-04-28T10:48:09.573927564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:48:35.423427051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51716"
ts=2026-04-28T10:48:39.576340899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:48:44.483742125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:48:49.148185995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46400"
ts=2026-04-28T10:49:09.57921525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:49:19.151542852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52000"
ts=2026-04-28T10:49:30.27204281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51910"
ts=2026-04-28T10:49:35.427378607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39810"
ts=2026-04-28T10:49:39.582667196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:49:44.048318956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:49:44.049866583Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:50:00.275023555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35802"
ts=2026-04-28T10:50:09.584648907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
level=debug ts=2026-04-28T10:50:19.466276571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:50:35.432039289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47854"
ts=2026-04-28T10:50:39.586051687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:50:49.159165801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34194"
ts=2026-04-28T10:51:05.433181928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34162"
ts=2026-04-28T10:51:09.58794155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:51:11.121638808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:51:35.434439328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45326"
ts=2026-04-28T10:51:39.589735046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:52:00.285266669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52080"
level=debug ts=2026-04-28T10:52:05.595456581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:52:09.59184084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:52:39.594390688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:52:49.171145019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44012"
level=debug ts=2026-04-28T10:52:56.139325844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:53:00.290125035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45778"
ts=2026-04-28T10:53:05.442618519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41708"
ts=2026-04-28T10:53:09.59628765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:53:19.17416232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40078"
ts=2026-04-28T10:53:30.29353368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36792"
level=debug ts=2026-04-28T10:53:32.38036416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:53:32.381945886Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:53:39.599607432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:53:49.179436653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34730"
ts=2026-04-28T10:54:09.602517127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:54:30.300325085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38004"
level=debug ts=2026-04-28T10:54:32.552693833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:54:35.44950358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45444"
ts=2026-04-28T10:54:39.605013849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:54:49.185641714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38330"
ts=2026-04-28T10:55:05.450953861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35550"
ts=2026-04-28T10:55:09.607395696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:55:25.386357583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:55:39.610766822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:55:49.19253046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49366"
level=debug ts=2026-04-28T10:56:03.606882049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:56:09.613671971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T10:56:19.195543043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35264"
ts=2026-04-28T10:56:39.616331656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T10:56:45.828934775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:56:49.19853162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44388"
ts=2026-04-28T10:57:05.460753827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50074"
ts=2026-04-28T10:57:09.619646012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:57:29.621116583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:57:29.62261466Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:57:30.317271714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59720"
ts=2026-04-28T10:57:39.622055773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
level=error ts=2026-04-28T10:57:42.768370066Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:57:42.770148384Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:57:42.77184159Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:57:42.771873063Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:58:09.624503392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T10:58:21.416823844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:58:39.626988444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T10:59:00.325202598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52012"
ts=2026-04-28T10:59:05.471537329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39580"
ts=2026-04-28T10:59:09.627950689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
level=debug ts=2026-04-28T10:59:18.609505453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:59:19.212319098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46452"
ts=2026-04-28T10:59:39.630010743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T10:59:49.215646088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44692"
ts=2026-04-28T11:00:00.330532559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46182"
level=debug ts=2026-04-28T11:00:09.516273352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:00:09.633066375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:00:39.636121243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:00:49.222079861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34674"
level=debug ts=2026-04-28T11:01:00.873721634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:01:09.639104464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:01:19.225680852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59998"
ts=2026-04-28T11:01:30.338437666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47428"
ts=2026-04-28T11:01:35.48472643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51002"
level=debug ts=2026-04-28T11:01:37.888529405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:01:37.890557915Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:01:39.641625572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e15f531d 10.130.0.72:7946"
ts=2026-04-28T11:01:49.22852516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40366"
ts=2026-04-28T11:02:00.340701128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46994"
ts=2026-04-28T11:02:05.486145958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44072"
ts=2026-04-28T11:02:09.644057213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:02:10.425088799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36650"
ts=2026-04-28T11:02:10.436165165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36660"
ts=2026-04-28T11:02:19.231605743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52982"
ts=2026-04-28T11:02:30.344036083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39466"
level=debug ts=2026-04-28T11:02:37.454093291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:02:39.647185616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:02:40.211676316Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e15f531d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T11:03:09.649829791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:03:19.237654623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:45228"
level=debug ts=2026-04-28T11:03:26.167756094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:03:30.350066755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49610"
ts=2026-04-28T11:03:39.650908234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:03:54.698555873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:58454"
level=debug ts=2026-04-28T11:04:00.5807466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:04:09.652939411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:04:19.244426775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48092"
ts=2026-04-28T11:04:30.355666343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:54488"
ts=2026-04-28T11:04:39.656087418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:04:44.013033826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:05:09.658331137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:05:23.347835265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:05:23.34968214Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:05:24.705832438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55770"
ts=2026-04-28T11:05:30.361118179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38418"
ts=2026-04-28T11:05:39.661171577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:05:57.847143702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:06:09.664635587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:06:39.667115731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:06:44.095183288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:06:49.260115404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:43240"
ts=2026-04-28T11:07:09.669053075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:07:25.519477966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:07:39.672241922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=error ts=2026-04-28T11:07:42.768524225Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:07:42.770385956Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:07:42.772036851Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:07:42.772064396Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:07:49.265836524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46296"
ts=2026-04-28T11:08:09.675563647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:08:18.751855617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:08:30.377092335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47354"
ts=2026-04-28T11:08:39.678733271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:08:49.271718691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:60970"
level=debug ts=2026-04-28T11:08:56.92282011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:08:56.924402567Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:09:00.380392714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49088"
ts=2026-04-28T11:09:09.681834436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:09:39.683691187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:09:40.172247595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:09:49.277233261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53070"
ts=2026-04-28T11:10:00.386208576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40244"
ts=2026-04-28T11:10:09.686186509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:10:24.734933649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53420"
level=debug ts=2026-04-28T11:10:25.113187814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:10:30.389022359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59454"
ts=2026-04-28T11:10:39.688283169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:10:54.73609519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34668"
ts=2026-04-28T11:11:09.691124223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:11:16.045770291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:11:19.285081695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54482"
ts=2026-04-28T11:11:39.69362488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:11:54.741099288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50654"
level=debug ts=2026-04-28T11:12:07.765455256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:12:09.694614568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:12:19.291249994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38896"
ts=2026-04-28T11:12:39.69688109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:12:42.24549158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:12:42.247069019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:13:09.700224236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:13:19.297132986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34462"
level=debug ts=2026-04-28T11:13:25.234201074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:13:39.702064807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:14:03.249503891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:14:09.70475588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:14:30.410773931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47142"
ts=2026-04-28T11:14:39.707960169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:14:46.733314069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:14:54.759023528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37800"
ts=2026-04-28T11:15:09.711207832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:15:19.308837955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48526"
level=debug ts=2026-04-28T11:15:39.311767256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:15:39.713073236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:16:09.715870747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:16:15.109709542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:16:15.111270711Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:16:39.718322884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:17:00.424728114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47852"
level=debug ts=2026-04-28T11:17:09.353600088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:17:09.71973837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:17:19.321182179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44294"
ts=2026-04-28T11:17:39.721903274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=error ts=2026-04-28T11:17:42.769298743Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:17:42.771244046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:17:42.772851813Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:17:42.772874286Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:17:54.776476033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47208"
level=debug ts=2026-04-28T11:18:03.42595619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:18:09.723878909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:18:39.726254303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:18:47.174786141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:19:09.729861648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:19:30.435678079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50022"
level=debug ts=2026-04-28T11:19:32.733425059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:19:39.733240642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
ts=2026-04-28T11:20:09.736424778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:20:31.843037772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:20:31.845073063Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:20:39.738931664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:21:09.740875869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:21:15.601853717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:21:24.798221486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53146"
ts=2026-04-28T11:21:30.445946128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43932"
ts=2026-04-28T11:21:39.742830809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:21:49.347291453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:49214"
level=debug ts=2026-04-28T11:21:50.340414479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:22:09.745253536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:22:36.913587934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:22:39.747746557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:22:49.352499383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:35908"
ts=2026-04-28T11:23:09.750471518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:23:24.808417199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48482"
level=debug ts=2026-04-28T11:23:29.890677713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:23:39.753130294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:24:00.458677393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:32838"
ts=2026-04-28T11:24:09.754802717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:24:16.537546913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:24:16.539451259Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:24:39.756425215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:24:53.517123148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:25:09.758760291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:25:37.694492202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:25:39.762189945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:25:49.371641183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59274"
ts=2026-04-28T11:26:09.76383234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
level=debug ts=2026-04-28T11:26:18.7008871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:26:19.374741928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55974"
ts=2026-04-28T11:26:24.826108494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36454"
ts=2026-04-28T11:26:39.765197399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7906467 10.130.0.72:7946"
ts=2026-04-28T11:27:00.472794947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51998"
ts=2026-04-28T11:27:09.767948364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-d8zls-954c0820 10.129.0.74:7946"
level=debug ts=2026-04-28T11:27:10.187601826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:27:39.771239287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=error ts=2026-04-28T11:27:42.769014313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:27:42.770854694Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:27:42.772672466Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:27:42.772695799Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:27:49.384959857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:57296"
ts=2026-04-28T11:27:54.833331342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41136"
level=debug ts=2026-04-28T11:28:03.17363086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:28:03.176144527Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:28:09.774755127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
ts=2026-04-28T11:28:19.388691673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42552"
ts=2026-04-28T11:28:24.834104955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41958"
ts=2026-04-28T11:28:39.778122774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-r7m6q-2b4378dc 10.128.0.66:7946"
level=debug ts=2026-04-28T11:28:43.223666537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"