level=info ts=2026-04-21T02:45:02.9251693Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T02:45:02.925199687Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T02:45:02.926412321Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T02:45:02.926849202Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T02:45:02.926955242Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.87 inf=eth0
level=debug ts=2026-04-21T02:45:02.928191361Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T02:45:02.928280264Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.87 inf=eth0
level=info ts=2026-04-21T02:45:02.928342149Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f8d18ebb
ts=2026-04-21T02:45:02.928435664Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T02:45:02.928450635Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.87 advertisePort=7946
level=debug ts=2026-04-21T02:45:02.928468938Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.87 advertisePort=7946
level=info ts=2026-04-21T02:45:02.928630212Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T02:45:02.934261492Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T02:45:02.934282461Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T02:45:02.93429953Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.671105ms
level=error ts=2026-04-21T02:45:02.934310096Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T02:45:02.934325267Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T02:45:02.937167904Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T02:45:02.937182663Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T02:45:02.942886757Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T02:45:02.942898677Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-21T02:45:02.942926504Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-21T02:45:02.942935755Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T02:45:02.94295331Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T02:45:02.942962633Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T02:45:02.943027085Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T02:45:02.943061779Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T02:45:02.943093277Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-21T02:45:02.943312794Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T02:45:02.943401214Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T02:45:02.943390174Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T02:45:02.943448786Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T02:45:02.943620995Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T02:45:02.94363056Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T02:45:02.943653585Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T02:45:03.944154108Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T02:45:04.059398477Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T02:45:04.059458655Z caller=loki.go:599 msg="Loki started" startup_time=1.211529462s
ts=2026-04-21T02:45:04.196559086Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T02:45:04.196586997Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:45:07.996003558Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T02:45:07.996028326Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T02:45:09.059716985Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T02:45:09.059771222Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T02:45:14.943086171Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T02:45:14.943111534Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T02:45:23.742910469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:45:24.74087672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.70:7946"
ts=2026-04-21T02:45:24.742709136Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.69:7946: dial tcp 10.128.0.69:7946: connect: connection refused"
ts=2026-04-21T02:45:24.743315751Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
level=info ts=2026-04-21T02:45:24.743343409Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=21.809011946s
level=debug ts=2026-04-21T02:45:46.670481074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:45:56.668004714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52450"
ts=2026-04-21T02:45:58.446353444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T02:46:07.255322568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:46:28.448551358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T02:46:32.114633551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34612"
level=debug ts=2026-04-21T02:46:32.568234249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:46:56.67209936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39472"
ts=2026-04-21T02:46:58.450602806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T02:47:00.499482393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T02:47:00.50131253Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T02:47:26.673838726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60502"
ts=2026-04-21T02:47:28.452789939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:47:37.529955137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51976"
level=debug ts=2026-04-21T02:47:39.575293375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:47:58.454724652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:48:28.455993734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:48:37.53296415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52826"
level=debug ts=2026-04-21T02:48:49.599271636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:48:58.457670557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:49:07.53584134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53972"
ts=2026-04-21T02:49:28.45957201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T02:49:48.436305735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:49:56.684495247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49218"
ts=2026-04-21T02:49:58.461142886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:50:02.127107554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59938"
ts=2026-04-21T02:50:28.462714848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:50:32.12871743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52000"
level=debug ts=2026-04-21T02:50:44.455178801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T02:50:58.464072756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T02:51:28.465525206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:51:32.132154531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57398"
level=debug ts=2026-04-21T02:51:34.550923773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T02:51:34.552679613Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T02:51:37.544160078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36490"
ts=2026-04-21T02:51:56.692538385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45542"
ts=2026-04-21T02:51:58.467749281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:52:07.545860394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37056"
ts=2026-04-21T02:52:26.694327048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59142"
ts=2026-04-21T02:52:28.469904639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T02:52:32.803978557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:52:58.471954113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:53:02.136625457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57780"
ts=2026-04-21T02:53:07.548759699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45200"
level=debug ts=2026-04-21T02:53:15.233998376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:53:28.473697834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T02:53:37.551142658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36942"
level=debug ts=2026-04-21T02:53:56.796772678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:53:58.478285146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:54:07.553253512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40904"
ts=2026-04-21T02:54:28.480185264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T02:54:36.356288847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:54:37.55473291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:49982"
ts=2026-04-21T02:54:56.705202449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41924"
ts=2026-04-21T02:54:58.481886507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=info ts=2026-04-21T02:55:09.059843813Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T02:55:09.065085812Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T02:55:09.067319498Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T02:55:09.069674188Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T02:55:09.069713383Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T02:55:09.069727372Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T02:55:28.483926073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T02:55:29.584173794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T02:55:29.586410236Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T02:55:37.557810804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:33836"
ts=2026-04-21T02:55:58.485381906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T02:56:06.353271296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:56:28.487353211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T02:56:32.147857567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56720"
ts=2026-04-21T02:56:37.561888735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45758"
level=debug ts=2026-04-21T02:56:51.727558997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:56:58.4887988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:57:28.490243852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T02:57:36.422294522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:57:58.491805389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:58:07.567828232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:35926"
level=debug ts=2026-04-21T02:58:19.329511337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:58:26.719868434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:35752"
ts=2026-04-21T02:58:28.49386111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T02:58:32.155710063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53622"
ts=2026-04-21T02:58:37.56969145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34132"
ts=2026-04-21T02:58:56.721684192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56754"
ts=2026-04-21T02:58:58.496131195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T02:58:58.64324667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T02:58:58.645677683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T02:59:07.572228255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:33496"
ts=2026-04-21T02:59:26.723404262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:47530"
ts=2026-04-21T02:59:28.497569346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T02:59:43.051672343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T02:59:56.724671445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54928"
ts=2026-04-21T02:59:58.499755217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:00:02.160437827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39508"
ts=2026-04-21T03:00:28.501084513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:00:28.811786966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:00:58.502959262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:01:19.067115983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:01:26.730458252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52866"
ts=2026-04-21T03:01:28.505383673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:01:32.16610223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54062"
ts=2026-04-21T03:01:58.507299313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:02:08.440782521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:02:28.509217522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:02:32.169635694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49156"
ts=2026-04-21T03:02:56.735312905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34164"
ts=2026-04-21T03:02:58.510419083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:03:08.333409112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:03:08.335096394Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:03:28.512303655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:03:50.17643312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:03:58.514499368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:04:28.516617883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:04:32.175737197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49478"
level=debug ts=2026-04-21T03:04:48.668623131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:04:56.742944341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45136"
ts=2026-04-21T03:04:58.518495236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=error ts=2026-04-21T03:05:09.073902973Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:05:09.076248743Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:05:09.078361217Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:05:09.078394798Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:05:28.520236531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:05:39.004072575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:05:58.521840443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:06:14.057389655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:06:26.748342739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48932"
ts=2026-04-21T03:06:28.524119791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:06:56.749658833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50772"
ts=2026-04-21T03:06:58.525673518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:07:03.450669873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:07:03.452375523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:07:07.598881641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51250"
ts=2026-04-21T03:07:28.527844086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:07:37.601106095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59042"
ts=2026-04-21T03:07:58.529516675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:08:02.277622422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:08:28.530958143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:08:46.615687156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:08:58.532288466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:09:02.19403201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42200"
level=debug ts=2026-04-21T03:09:23.747314017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:09:28.533644746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:09:37.607782304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36142"
ts=2026-04-21T03:09:56.762351199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:38024"
ts=2026-04-21T03:09:58.53555061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:10:13.826953481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:10:26.763576326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41950"
ts=2026-04-21T03:10:28.537380579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:10:58.539109413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:11:07.612208257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43416"
level=debug ts=2026-04-21T03:11:12.407110865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:11:12.409383107Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:11:28.541547984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:11:46.600719174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:11:56.769124604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44014"
ts=2026-04-21T03:11:58.543035033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:12:28.545168346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:12:32.204743875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51526"
level=debug ts=2026-04-21T03:12:36.660642453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:12:58.547178881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:13:02.206700727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37684"
ts=2026-04-21T03:13:26.776796354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53626"
ts=2026-04-21T03:13:28.548985631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T03:13:30.086706228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:13:58.553660511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:14:22.303894123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:14:26.780276201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40660"
ts=2026-04-21T03:14:28.556496399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:14:32.211858873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52650"
ts=2026-04-21T03:14:37.625116555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56202"
ts=2026-04-21T03:14:58.55808098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:15:02.213699405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38916"
level=debug ts=2026-04-21T03:15:09.048978925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:15:09.050646898Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:15:09.071956111Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:15:09.073659019Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:15:09.07557788Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:15:09.075622029Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:15:26.78425877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:55604"
ts=2026-04-21T03:15:28.559762858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:15:32.21576436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60044"
ts=2026-04-21T03:15:37.628609381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56636"
ts=2026-04-21T03:15:56.785500884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43390"
ts=2026-04-21T03:15:58.561109244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:16:00.057289632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:16:02.218143474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47904"
ts=2026-04-21T03:16:28.562476124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:16:32.220318516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47768"
level=debug ts=2026-04-21T03:16:39.770991098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:16:58.564108576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:17:07.633034138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40804"
ts=2026-04-21T03:17:26.791990398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42086"
ts=2026-04-21T03:17:28.565763222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:17:38.218206309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:17:56.793123086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58394"
ts=2026-04-21T03:17:58.567495922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:18:26.794460405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40000"
level=debug ts=2026-04-21T03:18:27.143876532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:18:28.569914797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:18:37.637956676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47978"
ts=2026-04-21T03:18:58.571791707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T03:19:10.258766017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:19:10.260389053Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:19:28.573718712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:19:37.64264433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57662"
level=debug ts=2026-04-21T03:19:52.655892131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:19:58.575502028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:20:28.576966653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:20:37.646592446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:60654"
level=debug ts=2026-04-21T03:20:46.937495815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:20:56.80459086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50972"
ts=2026-04-21T03:20:58.578379634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:21:07.649120992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36518"
ts=2026-04-21T03:21:26.806024189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42976"
level=debug ts=2026-04-21T03:21:27.715683833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:21:28.57996397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:21:56.807455921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39660"
ts=2026-04-21T03:21:58.582011748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:22:26.809107646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60652"
level=debug ts=2026-04-21T03:22:26.906699642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:22:28.584269935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:22:58.585649999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:23:02.243000252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43222"
level=debug ts=2026-04-21T03:23:03.361885923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:23:03.363405039Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:23:07.65566489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50762"
ts=2026-04-21T03:23:28.586746523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:23:54.384535325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:23:56.814491325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40732"
ts=2026-04-21T03:23:58.588029571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:24:28.590246357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:24:32.248762844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58478"
level=debug ts=2026-04-21T03:24:43.408300915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:24:56.818576164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:53628"
ts=2026-04-21T03:24:58.592878553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=error ts=2026-04-21T03:25:09.073357469Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:25:09.075312715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:25:09.076942453Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:25:09.07697213Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:25:26.820198869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46992"
ts=2026-04-21T03:25:28.594212499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:25:36.131450819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:25:58.59547139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:26:28.597242041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
level=debug ts=2026-04-21T03:26:34.426689057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:26:58.599316693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:27:10.864011429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:27:10.865617138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:27:28.601765964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:27:37.670349118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59298"
level=debug ts=2026-04-21T03:27:53.177167302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:27:58.603739673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:28:02.260123415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38874"
ts=2026-04-21T03:28:28.606220953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T03:28:46.740473281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:28:58.608239536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:29:02.26339962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42700"
ts=2026-04-21T03:29:28.610880396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T03:29:31.315882688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:29:32.264969981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45330"
ts=2026-04-21T03:29:37.679009717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53418"
ts=2026-04-21T03:29:58.612900149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:30:07.681088935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50832"
level=debug ts=2026-04-21T03:30:23.138227131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:30:28.614722577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:30:37.683403509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40862"
ts=2026-04-21T03:30:56.844173993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52020"
ts=2026-04-21T03:30:58.616876766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:31:02.270707971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52826"
level=debug ts=2026-04-21T03:31:02.540795431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:31:02.542426302Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:31:07.685732393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:35372"
ts=2026-04-21T03:31:28.619079142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:31:43.468342025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:31:58.621586558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:32:02.275779417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38040"
ts=2026-04-21T03:32:26.850963649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33660"
ts=2026-04-21T03:32:28.623672104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:32:37.69139418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47404"
level=debug ts=2026-04-21T03:32:43.068535653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:32:58.625431607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
level=debug ts=2026-04-21T03:33:22.880567379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:33:28.627587644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:33:58.629924733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
level=debug ts=2026-04-21T03:34:07.220211117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:34:28.631986256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2e457880 10.128.0.72:7946"
ts=2026-04-21T03:34:32.285680351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51618"
ts=2026-04-21T03:34:37.699893501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57764"
level=debug ts=2026-04-21T03:34:57.995117979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T03:34:57.996857977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:34:58.633199344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:35:07.701960447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44000"
level=error ts=2026-04-21T03:35:09.075762127Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:35:09.07793022Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:35:09.079915488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T03:35:09.079951425Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T03:35:28.634217134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-7kkd7-cbb042e2 10.129.0.85:7946"
ts=2026-04-21T03:35:37.703708368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46638"
level=debug ts=2026-04-21T03:35:41.17294357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T03:35:56.865799219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:41592"
ts=2026-04-21T03:35:58.635961433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-q7qng-10400b2f 10.128.0.70:7946"
ts=2026-04-21T03:36:07.705329172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57746"