level=info ts=2026-04-17T18:20:51.987196155Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T18:20:51.987221804Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-17T18:20:51.988552875Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T18:20:51.988873673Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T18:20:51.988972222Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.64 inf=eth0
level=debug ts=2026-04-17T18:20:51.989536241Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=info ts=2026-04-17T18:20:51.989654005Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-65bf025e
level=debug ts=2026-04-17T18:20:51.989657751Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.64 inf=eth0
ts=2026-04-17T18:20:51.989729224Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T18:20:51.989746503Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.64 advertisePort=7946
level=debug ts=2026-04-17T18:20:51.989764562Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.64 advertisePort=7946
level=info ts=2026-04-17T18:20:51.98990295Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-17T18:20:51.994869343Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T18:20:51.994890479Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T18:20:51.994905097Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.004354ms
level=error ts=2026-04-17T18:20:51.994911429Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T18:20:51.994921746Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-17T18:20:51.997240996Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T18:20:51.997252721Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T18:20:52.002416193Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-17T18:20:52.002436603Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-17T18:20:52.002440167Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-17T18:20:52.002440546Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-17T18:20:52.00245744Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-17T18:20:52.002472838Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-17T18:20:52.002558629Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-17T18:20:52.002572612Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-17T18:20:52.002712913Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-17T18:20:52.002741263Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-17T18:20:52.002859667Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-17T18:20:52.002885243Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T18:20:52.002919252Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T18:20:52.002992242Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-17T18:20:52.123230133Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-17T18:20:52.123301993Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T18:20:53.124058737Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-17T18:20:53.251887467Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-17T18:20:53.251934604Z caller=loki.go:599 msg="Loki started" startup_time=1.325235117s
ts=2026-04-17T18:20:53.753802067Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T18:20:53.753830073Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:20:57.748691527Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T18:20:57.748723608Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T18:20:58.252113383Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-17T18:20:58.252162027Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-17T18:21:03.941334733Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T18:21:03.941359503Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:21:11.406016691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:52730"
level=debug ts=2026-04-17T18:21:13.432954644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:21:14.425867814Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.66:7946: dial tcp 10.128.0.66:7946: connect: connection refused"
ts=2026-04-17T18:21:14.42599153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.64:7946"
ts=2026-04-17T18:21:14.426012268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.64:54686"
level=info ts=2026-04-17T18:21:14.426274503Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.431348495s
ts=2026-04-17T18:21:16.384300578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60308"
ts=2026-04-17T18:21:17.80252456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60748"
level=debug ts=2026-04-17T18:21:28.663266386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:21:41.054375774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35536"
ts=2026-04-17T18:21:44.296848604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:21:46.652464607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:21:58.155736549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:50320"
level=debug ts=2026-04-17T18:22:09.373174758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:22:14.299111962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:22:28.303774939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T18:22:28.305697281Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:22:28.542524057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:51168"
ts=2026-04-17T18:22:44.3002999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:22:58.54412211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44712"
level=debug ts=2026-04-17T18:23:14.123171956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:23:14.301928574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:23:44.304343911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:24:13.1308799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:24:14.306481465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:24:28.168565576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:39500"
ts=2026-04-17T18:24:44.307491537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:24:58.55339072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58666"
ts=2026-04-17T18:25:11.076006769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50854"
level=debug ts=2026-04-17T18:25:11.345946828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:25:14.309612878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:25:44.311707515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:26:11.082004045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42766"
ts=2026-04-17T18:26:14.314060512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:26:28.181728144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:46518"
ts=2026-04-17T18:26:28.559633559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44474"
level=debug ts=2026-04-17T18:26:30.025242692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T18:26:41.084527878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60012"
ts=2026-04-17T18:26:44.316336596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:27:14.318662812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:27:28.186534184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:58376"
level=debug ts=2026-04-17T18:27:29.220481367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T18:27:29.222078576Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:27:44.320468021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:28:14.321708479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:28:23.694706333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:28:28.190998222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:43854"
ts=2026-04-17T18:28:44.323411164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:29:00.707026912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:29:14.327161602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:29:28.196482252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:60522"
ts=2026-04-17T18:29:28.571417104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:54558"
ts=2026-04-17T18:29:44.32938994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:29:52.251357771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:29:58.572728742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:37194"
ts=2026-04-17T18:30:14.330950614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:30:41.570282433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:30:44.33281009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=info ts=2026-04-17T18:30:58.252494237Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-17T18:30:58.257273021Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:30:58.259134669Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:30:58.260753074Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:30:58.260779369Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T18:30:58.26079069Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-17T18:31:14.334828127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:31:28.579567141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:38450"
level=debug ts=2026-04-17T18:31:39.450093673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:31:39.451728412Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:31:44.336939358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:31:58.212354694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:34026"
ts=2026-04-17T18:32:14.338974051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:32:24.105833822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:32:28.584408496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:57368"
ts=2026-04-17T18:32:41.124873802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59284"
ts=2026-04-17T18:32:44.341877371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:32:58.216431878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:60204"
ts=2026-04-17T18:32:58.585192802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:50620"
level=debug ts=2026-04-17T18:33:08.506933825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:33:11.127387091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44222"
ts=2026-04-17T18:33:14.344380936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:33:28.586573043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40126"
ts=2026-04-17T18:33:44.346654051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:33:55.774379485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:33:58.222713629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:50678"
ts=2026-04-17T18:33:58.587891485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48018"
ts=2026-04-17T18:34:11.132977201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33044"
ts=2026-04-17T18:34:14.349030226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:34:28.589118161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:41396"
level=debug ts=2026-04-17T18:34:40.113479113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:34:44.351800491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:34:58.2282515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:43032"
ts=2026-04-17T18:35:14.353814468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:35:16.925611511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:35:16.927207303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:35:28.230800558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:58362"
ts=2026-04-17T18:35:44.355562181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:36:14.358087432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:36:16.010823011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:36:41.14833694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52552"
ts=2026-04-17T18:36:44.360890453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:36:58.993386567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:37:14.363360109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:37:28.239775028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:41888"
level=debug ts=2026-04-17T18:37:35.73381333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:37:44.365930633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:38:14.368455976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:38:31.307691775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:38:41.160933031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49360"
ts=2026-04-17T18:38:44.37091546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:39:11.163889844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35756"
level=debug ts=2026-04-17T18:39:13.710149422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:39:13.711857963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:39:14.373240074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:39:28.250620681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:59244"
ts=2026-04-17T18:39:44.375709537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:39:50.773335178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:40:14.378845191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:40:28.255737222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:42430"
level=debug ts=2026-04-17T18:40:36.730129555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:40:44.380812682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:40:58.257779963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:46190"
level=error ts=2026-04-17T18:40:58.264727696Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:40:58.266541798Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:40:58.268519827Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:40:58.268544136Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:40:58.624784834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40702"
ts=2026-04-17T18:41:11.176101895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48960"
ts=2026-04-17T18:41:14.382569924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:41:28.260274323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:54272"
level=debug ts=2026-04-17T18:41:38.893841759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:41:44.38476063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:42:11.181929088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51704"
ts=2026-04-17T18:42:14.386407204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:42:32.754966425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:42:44.388382773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:42:58.633114226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:41840"
ts=2026-04-17T18:43:11.18821975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40518"
ts=2026-04-17T18:43:14.391034012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:43:20.399938557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:43:20.401453645Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:43:28.270886643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:38734"
ts=2026-04-17T18:43:28.634319717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49116"
ts=2026-04-17T18:43:41.191003798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46434"
ts=2026-04-17T18:43:44.392328933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:43:58.988328138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:44:11.193597161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53678"
ts=2026-04-17T18:44:14.394132184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:44:36.464178414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:44:44.396772932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:45:14.398820841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:45:34.418725083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:45:41.202750706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34114"
ts=2026-04-17T18:45:44.401099081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:46:11.205759978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46920"
ts=2026-04-17T18:46:14.403683218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:46:33.738945835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:46:44.406364545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:46:58.652471152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49506"
level=debug ts=2026-04-17T18:47:12.473154788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:47:12.474886122Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:47:14.409803896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:47:28.294273401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:53286"
ts=2026-04-17T18:47:44.412132926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:47:53.968746128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:48:14.413525543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:48:42.952726512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:48:44.415887111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:49:14.418243272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:49:41.225920889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51842"
ts=2026-04-17T18:49:44.420657627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:49:44.658421523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:49:58.666294887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34848"
ts=2026-04-17T18:50:14.42289324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:50:28.310100711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:32788"
ts=2026-04-17T18:50:44.425345908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:50:44.608450921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:50:58.264477363Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:50:58.266204733Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:50:58.267915179Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T18:50:58.267939893Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:51:14.427770506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:51:23.866658399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:51:23.86811133Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:51:28.672698889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40378"
ts=2026-04-17T18:51:44.430877942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:52:04.715477994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:52:14.43359252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:52:28.322295288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:42232"
ts=2026-04-17T18:52:44.436077136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:53:02.799212271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:53:14.438511925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:53:28.32683541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:40002"
ts=2026-04-17T18:53:44.441048357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:53:58.329095576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:53484"
level=debug ts=2026-04-17T18:54:04.792205842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:54:14.443199757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:54:40.620004339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:54:44.446104496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:54:58.334624578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:50748"
ts=2026-04-17T18:55:14.448904532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
level=debug ts=2026-04-17T18:55:25.37722853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:55:25.379271112Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:55:28.336639277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:53558"
ts=2026-04-17T18:55:28.690750845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:42424"
ts=2026-04-17T18:55:41.264594969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56132"
ts=2026-04-17T18:55:44.451719119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:55:58.338903347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:50086"
ts=2026-04-17T18:55:58.691874426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49230"
ts=2026-04-17T18:56:11.26809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54334"
ts=2026-04-17T18:56:14.453830914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T18:56:20.18674457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:56:28.340854939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:39448"
ts=2026-04-17T18:56:28.693714687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58114"
ts=2026-04-17T18:56:41.271085055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42792"
ts=2026-04-17T18:56:44.455922529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T18:56:58.342748981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:52354"
ts=2026-04-17T18:57:11.27404187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47480"
ts=2026-04-17T18:57:14.458921198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:57:18.248942459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:57:44.460757859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:57:58.346901303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:56702"
ts=2026-04-17T18:57:58.701134444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:33984"
level=debug ts=2026-04-17T18:58:00.427673676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:58:11.280619892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55464"
ts=2026-04-17T18:58:14.463022239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:58:28.349254961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:33066"
ts=2026-04-17T18:58:28.702772085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52662"
level=debug ts=2026-04-17T18:58:39.352771051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T18:58:41.283114009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57142"
ts=2026-04-17T18:58:44.465502195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T18:58:58.35126683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:39000"
ts=2026-04-17T18:59:14.467819688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T18:59:41.10667763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T18:59:41.10884826Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T18:59:44.469657986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T18:59:58.708482112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45692"
ts=2026-04-17T19:00:14.471897542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T19:00:18.041552164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:00:44.473225795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=error ts=2026-04-17T19:00:58.264411022Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T19:00:58.266235282Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T19:00:58.267901043Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-17T19:00:58.267923212Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T19:00:58.712912189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52330"
level=debug ts=2026-04-17T19:01:09.150478382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:01:11.306151821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35338"
ts=2026-04-17T19:01:14.478744093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T19:01:28.714258734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59428"
ts=2026-04-17T19:01:44.48232936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T19:01:58.368373468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:53790"
ts=2026-04-17T19:01:58.715040528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35644"
level=debug ts=2026-04-17T19:02:03.653788985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:02:11.311597712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54618"
ts=2026-04-17T19:02:14.485495554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T19:02:28.369930993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:58400"
ts=2026-04-17T19:02:41.314360051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58846"
ts=2026-04-17T19:02:44.488465043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T19:02:58.718541341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55462"
level=debug ts=2026-04-17T19:03:02.612474373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:03:14.490393252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T19:03:28.374691409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:39782"
ts=2026-04-17T19:03:44.492100789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T19:03:49.942412513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T19:03:49.944138104Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T19:03:58.722141417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:46428"
ts=2026-04-17T19:04:14.493891936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T19:04:25.633509756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:04:41.32598193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43914"
ts=2026-04-17T19:04:44.495714607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T19:04:58.382838767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:48284"
level=debug ts=2026-04-17T19:05:08.229642238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:05:14.497342868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T19:05:28.728386833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55316"
ts=2026-04-17T19:05:44.501944046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T19:05:56.443489534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:05:58.388446938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:36056"
ts=2026-04-17T19:05:58.72942248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34480"
ts=2026-04-17T19:06:14.502810972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T19:06:28.390356497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.85:48732"
ts=2026-04-17T19:06:44.504444191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
level=debug ts=2026-04-17T19:06:46.390211729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:07:14.508727869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gfr2c-60f41172 10.129.0.81:7946"
ts=2026-04-17T19:07:28.735787642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58870"
level=debug ts=2026-04-17T19:07:43.646713931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T19:07:43.648226084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T19:07:44.511407588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T19:07:58.736799388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52048"
ts=2026-04-17T19:08:14.51334121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
ts=2026-04-17T19:08:28.738430994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:41400"
level=debug ts=2026-04-17T19:08:32.040059081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T19:08:44.514802065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-mtcq7-9c2c38d0 10.130.0.85:7946"
ts=2026-04-17T19:09:11.354747216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34174"
ts=2026-04-17T19:09:14.516163176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1f22ebab 10.128.0.65:7946"
level=debug ts=2026-04-17T19:09:27.386172813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"