level=info ts=2026-04-18T06:22:57.533435606Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-18T06:22:57.533463813Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-18T06:22:57.534519975Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-18T06:22:57.534835812Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-18T06:22:57.534920189Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.59 inf=eth0
level=debug ts=2026-04-18T06:22:57.53560204Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-18T06:22:57.535719553Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.59 inf=eth0
level=info ts=2026-04-18T06:22:57.535745166Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-4e4bfab5
ts=2026-04-18T06:22:57.535808163Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-18T06:22:57.535821011Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.59 advertisePort=7946
level=debug ts=2026-04-18T06:22:57.535835045Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.59 advertisePort=7946
level=info ts=2026-04-18T06:22:57.535940891Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-18T06:22:57.542188472Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-18T06:22:57.542206612Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-18T06:22:57.542215441Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.276811ms
level=error ts=2026-04-18T06:22:57.542222952Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-18T06:22:57.542233147Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-18T06:22:57.544653859Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-18T06:22:57.544678955Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-18T06:22:57.550338924Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-18T06:22:57.550364722Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-18T06:22:57.550356166Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-18T06:22:57.550363132Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-18T06:22:57.550384239Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-18T06:22:57.550401303Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-18T06:22:57.550421272Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-18T06:22:57.550490377Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-18T06:22:57.550505357Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-18T06:22:57.550674106Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-18T06:22:57.550722115Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-18T06:22:57.550738099Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-18T06:22:57.550769121Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-18T06:22:57.550927262Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-18T06:22:57.550941565Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-18T06:22:57.550972175Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-18T06:22:58.551456029Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-18T06:22:58.7056517Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-18T06:22:58.705706932Z caller=loki.go:599 msg="Loki started" startup_time=1.228827454s
ts=2026-04-18T06:22:59.484848179Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-18T06:22:59.484872609Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:23:01.943835257Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-18T06:23:01.943862508Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-18T06:23:03.706017187Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-18T06:23:03.706070437Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-18T06:23:06.565012103Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-18T06:23:06.565037189Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:23:15.710290508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.59:7946"
ts=2026-04-18T06:23:15.710326659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36138"
ts=2026-04-18T06:23:15.711569358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.58:7946"
ts=2026-04-18T06:23:15.713915611Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
ts=2026-04-18T06:23:15.715112635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.77:7946"
ts=2026-04-18T06:23:15.71761437Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
level=info ts=2026-04-18T06:23:15.717644789Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=18.175407143s
level=debug ts=2026-04-18T06:23:16.296728221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:23:18.064570425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40176"
ts=2026-04-18T06:23:21.065543652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33784"
ts=2026-04-18T06:23:21.34990423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:41372"
ts=2026-04-18T06:23:32.322165221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:23:34.300432092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-18T06:23:50.050951386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:23:50.878804522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:35906"
ts=2026-04-18T06:23:51.687175886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:55006"
ts=2026-04-18T06:23:53.389938202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:32954"
ts=2026-04-18T06:24:02.325435926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:24:12.545435446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:24:20.881089157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53628"
ts=2026-04-18T06:24:23.39255202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:41714"
ts=2026-04-18T06:24:32.328762893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:24:42.749941291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-18T06:24:42.751466618Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:25:02.331753444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:25:21.693740283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:56582"
level=debug ts=2026-04-18T06:25:31.62091535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:25:32.333187547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:25:50.888342791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58166"
ts=2026-04-18T06:26:02.335198795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:26:21.697934402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:49786"
level=debug ts=2026-04-18T06:26:28.827173937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:26:32.33784043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:26:50.894019612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:36820"
ts=2026-04-18T06:27:02.339530081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:27:21.70219385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:43524"
ts=2026-04-18T06:27:32.341861452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:27:34.681031106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:27:53.413423513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52714"
ts=2026-04-18T06:28:02.344047366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:28:20.900488584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47794"
level=debug ts=2026-04-18T06:28:24.514058275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-18T06:28:32.346200652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:29:02.348093409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:29:29.392839017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-18T06:29:29.394372566Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:29:32.350792696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:30:02.354216022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:30:15.558391657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:30:32.357172052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:30:50.910978166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52904"
ts=2026-04-18T06:30:51.719777162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:46794"
level=debug ts=2026-04-18T06:30:53.858609409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:31:02.360241261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:31:28.177569855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:31:32.362291881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:31:50.916136993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:47242"
ts=2026-04-18T06:32:02.364848529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:32:07.966061495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:32:21.725982488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:46720"
ts=2026-04-18T06:32:32.367513769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:32:50.920457168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34030"
ts=2026-04-18T06:32:51.727295517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:59220"
ts=2026-04-18T06:33:02.370008234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=info ts=2026-04-18T06:33:03.706430186Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-18T06:33:03.709010021Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:33:03.711028691Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:33:03.71285978Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:33:03.712885906Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-18T06:33:03.712898375Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-18T06:33:05.445255845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:33:05.447081276Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:33:21.72886984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:33332"
ts=2026-04-18T06:33:23.447097811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39312"
ts=2026-04-18T06:33:32.373059551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:33:49.195507622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:33:51.730477534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:55478"
ts=2026-04-18T06:34:02.375572106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:34:23.452886923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59168"
level=debug ts=2026-04-18T06:34:29.699418947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:34:32.377846111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:34:50.929985935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:43682"
ts=2026-04-18T06:34:53.455283512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50096"
ts=2026-04-18T06:35:02.381110617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:35:19.69150116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:35:32.384254058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:35:57.430659112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:36:02.385955163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:36:21.740829913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:44418"
ts=2026-04-18T06:36:32.388254796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:36:48.080514803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:36:48.082298343Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:37:02.391201885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:37:20.940497853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:52828"
ts=2026-04-18T06:37:21.745217694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:47374"
ts=2026-04-18T06:37:32.392872671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:37:40.54970976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:38:02.3946437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:38:20.94551951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:56228"
ts=2026-04-18T06:38:23.478551696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33252"
ts=2026-04-18T06:38:32.397893735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:38:39.492691416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:38:53.481707472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57892"
ts=2026-04-18T06:39:02.401246818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:39:32.403518919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:39:38.859459741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:39:50.961053677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:53306"
ts=2026-04-18T06:40:02.406322032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:40:20.963782666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:55198"
ts=2026-04-18T06:40:23.490308233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36606"
ts=2026-04-18T06:40:32.409688246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:40:32.637958002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:40:51.766402801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:54340"
ts=2026-04-18T06:41:02.411897433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:41:13.729267736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:41:13.73114054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:41:21.767418266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:42150"
ts=2026-04-18T06:41:32.413685983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:41:48.353561369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:42:02.416418522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:42:21.77061772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:51310"
ts=2026-04-18T06:42:32.418860719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:42:40.442918121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:42:53.506186483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52164"
ts=2026-04-18T06:43:02.420911335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=error ts=2026-04-18T06:43:03.717605355Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:43:03.71983739Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:43:03.721762648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:43:03.721789013Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-18T06:43:31.073051955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:43:32.42377866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:43:50.983683535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:38554"
ts=2026-04-18T06:43:53.511840595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50476"
ts=2026-04-18T06:44:02.426059214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:44:17.447661456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:44:32.42850381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:44:50.988831011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:36658"
level=debug ts=2026-04-18T06:44:52.353176499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:44:52.354768628Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:45:02.430427046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:45:29.19961311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:45:32.431861192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:46:02.433863385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:46:15.29353996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:46:20.998069437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:40718"
ts=2026-04-18T06:46:23.528140753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44220"
ts=2026-04-18T06:46:32.436337248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:46:51.792857439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:38000"
ts=2026-04-18T06:46:53.530729036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:60364"
ts=2026-04-18T06:47:02.438499514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:47:05.216610607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:47:32.441536589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:47:51.005059465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:42776"
ts=2026-04-18T06:48:02.443931578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:48:02.461476684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:48:21.007382989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:33688"
ts=2026-04-18T06:48:23.539268875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49314"
ts=2026-04-18T06:48:32.445757785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:48:41.360935313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:48:41.362879441Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:48:51.008920162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:39394"
ts=2026-04-18T06:48:51.802941258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:34760"
ts=2026-04-18T06:49:02.447660144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:49:23.545208218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40870"
level=debug ts=2026-04-18T06:49:29.92630933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:49:32.449578535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:49:51.014178577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:41822"
ts=2026-04-18T06:50:02.451359887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:50:20.993868424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:50:32.453507178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:50:51.811473944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:52982"
ts=2026-04-18T06:51:02.455902883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:51:05.003045053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:51:23.556798697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51238"
ts=2026-04-18T06:51:32.458078465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:51:44.552139182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:52:02.460587882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T06:52:32.462767588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:52:43.680706696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:52:43.682374908Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:52:51.81965597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:35504"
ts=2026-04-18T06:53:02.465403334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=error ts=2026-04-18T06:53:03.716959232Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:53:03.718979455Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:53:03.720815351Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T06:53:03.720844666Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:53:32.468392168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:53:45.66915718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:54:02.47146439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:54:21.825282047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:57282"
ts=2026-04-18T06:54:32.472827757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:54:36.671784093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:54:51.826128827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:33034"
ts=2026-04-18T06:54:53.577790426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39892"
ts=2026-04-18T06:55:02.474946453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:55:16.198902376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:55:21.827730317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:48992"
ts=2026-04-18T06:55:32.47701906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:56:02.47848544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T06:56:17.298442965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:56:32.481016003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T06:56:51.053259209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:59722"
ts=2026-04-18T06:57:02.484364967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:57:19.28781443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T06:57:19.289740003Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T06:57:21.055839449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58372"
ts=2026-04-18T06:57:21.839223961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:59492"
ts=2026-04-18T06:57:32.486650583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:58:02.489122369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T06:58:16.807304086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:58:21.061462106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:48972"
ts=2026-04-18T06:58:21.842405618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:57164"
ts=2026-04-18T06:58:23.599432362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51700"
ts=2026-04-18T06:58:32.492021831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:58:51.063671611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:37390"
ts=2026-04-18T06:58:51.843574558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:39550"
level=debug ts=2026-04-18T06:59:02.286874353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:59:02.494237837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T06:59:32.496347077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T06:59:49.571031108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T06:59:51.847039352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:53332"
ts=2026-04-18T07:00:02.498778775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:00:32.500806852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T07:00:51.047713269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:00:53.614128489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51718"
ts=2026-04-18T07:01:02.503122487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T07:01:21.07720099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:49586"
level=debug ts=2026-04-18T07:01:31.912756529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T07:01:31.914685308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T07:01:32.506715495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
ts=2026-04-18T07:01:51.858110798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:51878"
ts=2026-04-18T07:02:02.509036329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:02:32.512120543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T07:02:33.391499046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:02:53.62570712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53684"
ts=2026-04-18T07:03:02.514335199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=error ts=2026-04-18T07:03:03.719498818Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T07:03:03.721858501Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T07:03:03.723863858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-18T07:03:03.723900956Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-18T07:03:10.507079014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:03:21.087171857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:58876"
ts=2026-04-18T07:03:32.51670808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:03:51.089197489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:34526"
ts=2026-04-18T07:03:51.866763703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:43142"
ts=2026-04-18T07:04:02.519344181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T07:04:08.263702662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:04:23.634513659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49782"
ts=2026-04-18T07:04:32.522251818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:05:02.523813179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T07:05:04.855090937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:05:32.526011305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T07:05:59.421606308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T07:05:59.423467935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T07:06:02.528704214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:06:21.877312593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:57236"
ts=2026-04-18T07:06:32.531793474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-nsr6w-5cd89baf 10.129.0.77:7946"
level=debug ts=2026-04-18T07:06:43.162210796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:06:51.878808106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:37364"
ts=2026-04-18T07:06:53.650020207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38202"
ts=2026-04-18T07:07:02.53470483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
ts=2026-04-18T07:07:21.880472568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.58:32896"
level=debug ts=2026-04-18T07:07:23.72920914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:07:32.536486492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T07:07:53.655500856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38604"
ts=2026-04-18T07:08:02.537697477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T07:08:08.630392326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:08:21.110869089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:51282"
ts=2026-04-18T07:08:32.539546297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"
level=debug ts=2026-04-18T07:08:49.301029498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-18T07:08:51.112660584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.58:42936"
ts=2026-04-18T07:09:02.540817056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
ts=2026-04-18T07:09:32.542315434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ft6zd-0fccfc5d 10.130.0.58:7946"
level=debug ts=2026-04-18T07:09:35.203563505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-18T07:09:35.205475244Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-18T07:10:02.543848839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-15781284 10.128.0.58:7946"