level=info ts=2026-05-18T20:23:21.589341231Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)" level=info ts=2026-05-18T20:23:21.589393593Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml level=info ts=2026-05-18T20:23:21.591961319Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095 level=debug ts=2026-05-18T20:23:21.592404597Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-18T20:23:21.592496433Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.74 inf=eth0 level=debug ts=2026-05-18T20:23:21.593056726Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false level=debug ts=2026-05-18T20:23:21.593124646Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.74 inf=eth0 level=info ts=2026-05-18T20:23:21.5931644Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f628004e ts=2026-05-18T20:23:21.593251224Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired" level=debug ts=2026-05-18T20:23:21.593267346Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.74 advertisePort=7946 level=debug ts=2026-05-18T20:23:21.593293635Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.74 advertisePort=7946 level=info ts=2026-05-18T20:23:21.593462858Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4 ts=2026-05-18T20:23:21.599976176Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=info ts=2026-05-18T20:23:21.599996988Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=warn ts=2026-05-18T20:23:21.600010554Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.549673ms level=error ts=2026-05-18T20:23:21.600027377Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1" level=info ts=2026-05-18T20:23:21.600041577Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local ts=2026-05-18T20:23:21.602559755Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-18T20:23:21.602576482Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-05-18T20:23:21.609616986Z caller=module_service.go:82 msg=starting module=runtime-config level=debug ts=2026-05-18T20:23:21.609630915Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server level=debug ts=2026-05-18T20:23:21.609637781Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv level=info ts=2026-05-18T20:23:21.609650651Z caller=module_service.go:82 msg=starting module=server level=info ts=2026-05-18T20:23:21.609651805Z caller=module_service.go:82 msg=starting module=analytics level=info ts=2026-05-18T20:23:21.609683209Z caller=module_service.go:82 msg=starting module=memberlist-kv level=debug ts=2026-05-18T20:23:21.609703115Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config level=debug ts=2026-05-18T20:23:21.609860098Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server level=debug ts=2026-05-18T20:23:21.60987042Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics level=info ts=2026-05-18T20:23:21.609876328Z caller=module_service.go:82 msg=starting module=compactor level=info ts=2026-05-18T20:23:21.609953867Z caller=ring.go:390 msg="ring doesn't exist in KV store yet" level=info ts=2026-05-18T20:23:21.60994216Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor level=info ts=2026-05-18T20:23:21.609968421Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty" level=info ts=2026-05-18T20:23:21.610096835Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring" level=info ts=2026-05-18T20:23:21.61010644Z caller=compactor.go:416 msg="compactor is JOINING in the ring" level=debug ts=2026-05-18T20:23:21.610116779Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true level=info ts=2026-05-18T20:23:22.610617618Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring" level=info ts=2026-05-18T20:23:22.757898675Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring" level=info ts=2026-05-18T20:23:22.757962986Z caller=loki.go:615 msg="Loki started" startup_time=1.230846748s ts=2026-05-18T20:23:22.933704824Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-18T20:23:22.933728251Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:23:25.807069065Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-18T20:23:25.807092907Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" level=info ts=2026-05-18T20:23:27.758796863Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor" level=info ts=2026-05-18T20:23:27.758853079Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor" ts=2026-05-18T20:23:29.928197941Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host" level=warn ts=2026-05-18T20:23:29.92823186Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:23:39.176857858Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.69:7946: dial tcp 10.130.0.69:7946: connect: connection refused" ts=2026-05-18T20:23:39.178289328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.130.0.68:7946" ts=2026-05-18T20:23:39.180752453Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.73:7946: dial tcp 10.128.0.73:7946: connect: connection refused" ts=2026-05-18T20:23:39.181418358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.72:7946" ts=2026-05-18T20:23:39.18442543Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused" ts=2026-05-18T20:23:39.184542278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.128.0.74:7946" ts=2026-05-18T20:23:39.184558295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46248" level=info ts=2026-05-18T20:23:39.184846172Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=17.584797329s ts=2026-05-18T20:23:41.141084568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58124" ts=2026-05-18T20:23:41.436098049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39816" level=debug ts=2026-05-18T20:23:42.286055779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:23:47.487700158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42236" ts=2026-05-18T20:23:56.906952684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T20:23:59.2854359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:24:06.382768203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58150" level=debug ts=2026-05-18T20:24:19.209403499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:24:26.910190685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:24:36.576799693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:24:39.037728562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34408" ts=2026-05-18T20:24:47.205866396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54866" ts=2026-05-18T20:24:56.913412965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:24:59.824446852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-18T20:24:59.826082408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:25:26.916605812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:25:33.74874675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:25:47.211092868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55620" ts=2026-05-18T20:25:56.919297357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:26:26.921667386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:26:36.396153977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50708" level=debug ts=2026-05-18T20:26:40.499475039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:26:56.924872392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:27:06.398803309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41872" ts=2026-05-18T20:27:17.218816359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50912" ts=2026-05-18T20:27:26.927836201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:27:39.053508735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45328" level=debug ts=2026-05-18T20:27:41.81836502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:27:56.930965539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:28:09.055177125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39764" ts=2026-05-18T20:28:26.932788537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:28:47.227385164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33254" ts=2026-05-18T20:28:56.93405096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:28:59.839079647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" ts=2026-05-18T20:29:26.936189223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:29:56.938879443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T20:29:59.48259763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n" level=error ts=2026-05-18T20:29:59.484076134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:30:09.066770305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49968" ts=2026-05-18T20:30:17.234835608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48734" ts=2026-05-18T20:30:26.9415873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:30:47.237968966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57502" level=debug ts=2026-05-18T20:30:49.573223442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:30:56.944610325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:31:09.071216128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57126" ts=2026-05-18T20:31:17.240563219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46504" ts=2026-05-18T20:31:26.947505921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T20:31:29.065263228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:31:56.950813613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:32:09.07562614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37374" level=debug ts=2026-05-18T20:32:13.149417239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:32:26.953765408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:32:39.077011894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49640" ts=2026-05-18T20:32:56.955647371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:33:02.410115514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:33:17.249564368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37618" ts=2026-05-18T20:33:26.958169194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=info ts=2026-05-18T20:33:27.759590901Z caller=tables_manager.go:76 msg="compactor startup delay completed" level=error ts=2026-05-18T20:33:27.763291847Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:33:27.765448523Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:33:27.767281627Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:33:27.767308333Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=info ts=2026-05-18T20:33:27.767336202Z caller=tables_manager.go:148 msg="compactor started" ts=2026-05-18T20:33:36.432737124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39842" ts=2026-05-18T20:33:39.080902809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45884" level=debug ts=2026-05-18T20:33:41.355996827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:33:41.357758294Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:33:56.960489919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:34:26.962947328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:34:31.385606869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:34:56.966195844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:35:06.443668778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44746" level=debug ts=2026-05-18T20:35:19.888519197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:35:26.968644822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:35:36.446744389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58770" ts=2026-05-18T20:35:56.971234709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:36:06.450051864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35362" ts=2026-05-18T20:36:09.095214602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42082" level=debug ts=2026-05-18T20:36:11.119457284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:36:26.974082862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:36:47.268557663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55332" level=debug ts=2026-05-18T20:36:51.919454995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:36:56.976966429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:37:17.271823447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41084" ts=2026-05-18T20:37:26.980753858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:37:36.459861716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46032" level=debug ts=2026-05-18T20:37:50.089869473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:37:50.091459091Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:37:56.983799531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:38:26.987368343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:38:36.467344682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59580" level=debug ts=2026-05-18T20:38:46.592746045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:38:47.278981762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51428" ts=2026-05-18T20:38:56.990105993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:39:06.470467772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54558" ts=2026-05-18T20:39:26.991821642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:39:36.473377897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48448" level=debug ts=2026-05-18T20:39:37.721478501Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:39:56.994190579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:40:17.286349019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47846" ts=2026-05-18T20:40:26.997168358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:40:36.479953809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52894" level=debug ts=2026-05-18T20:40:39.060497882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:40:47.288929299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40830" ts=2026-05-18T20:40:57.000673338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:41:06.482926876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50058" ts=2026-05-18T20:41:27.003493775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:41:36.485623082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38438" level=debug ts=2026-05-18T20:41:37.03395338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:41:57.006893918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:42:09.129923514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55206" ts=2026-05-18T20:42:17.298055395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37136" ts=2026-05-18T20:42:27.00909145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:42:28.843167665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:42:28.845441462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:42:36.490853435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54790" ts=2026-05-18T20:42:47.301247236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45498" ts=2026-05-18T20:42:57.011544692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:43:07.262351513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:43:27.014711485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=error ts=2026-05-18T20:43:27.771339687Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:43:27.772981462Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:43:27.774551408Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:43:27.774571927Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:43:36.496889957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56672" ts=2026-05-18T20:43:39.139227915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51304" level=debug ts=2026-05-18T20:43:44.026906232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:43:57.017705526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:44:06.500479175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51234" ts=2026-05-18T20:44:17.310340151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46316" ts=2026-05-18T20:44:27.019991906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:44:32.927217144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:44:39.143702979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59958" ts=2026-05-18T20:44:57.021615303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:45:18.382137889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:45:27.023532709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:45:47.318739863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57420" ts=2026-05-18T20:45:57.026104342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:46:13.623664196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:46:13.62518986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:46:27.027806838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:46:57.030107133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T20:47:09.705504474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:47:27.032187042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:47:39.160737151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46692" ts=2026-05-18T20:47:57.03424319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:48:07.070074919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:48:27.037111038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:48:36.52527612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49692" ts=2026-05-18T20:48:47.334255363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58102" ts=2026-05-18T20:48:57.039687073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:49:06.09038659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:49:06.528049233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36790" ts=2026-05-18T20:49:17.336967591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36528" ts=2026-05-18T20:49:27.04068007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:49:49.216236263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:49:57.043497703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:50:06.534009904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33390" ts=2026-05-18T20:50:09.17483838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37730" ts=2026-05-18T20:50:27.045357676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:50:31.817622239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:50:31.819251622Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:50:57.047519983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:51:09.178747435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33992" level=debug ts=2026-05-18T20:51:11.193314244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:51:27.050464135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:51:47.350693786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43042" level=debug ts=2026-05-18T20:51:48.351199841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:51:57.053350043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:52:06.546796222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41096" ts=2026-05-18T20:52:27.055354092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:52:27.202685104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:52:39.186452307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53476" ts=2026-05-18T20:52:57.056235638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:53:17.358678196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52336" level=debug ts=2026-05-18T20:53:26.479075136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:53:27.059092476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=error ts=2026-05-18T20:53:27.769938509Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:53:27.771553752Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:53:27.773193987Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T20:53:27.773216854Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:53:57.061891622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:54:11.247744342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:54:11.249292659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:54:27.065610048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:54:57.069099915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:55:10.66329385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:55:27.070945304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:55:51.073692701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:55:57.072871364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:56:06.571222982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56538" level=debug ts=2026-05-18T20:56:25.294667274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:56:27.075916016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T20:56:39.209256655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42622" ts=2026-05-18T20:56:57.078274387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:57:24.450367946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:57:27.079472955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T20:57:36.580731817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59752" ts=2026-05-18T20:57:47.382297836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53326" ts=2026-05-18T20:57:57.081451831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:58:06.584525409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50180" level=debug ts=2026-05-18T20:58:07.492551603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T20:58:07.494097832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T20:58:27.084733861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T20:58:39.220175434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58134" ts=2026-05-18T20:58:47.38767393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41200" ts=2026-05-18T20:58:57.087360307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T20:58:57.968452666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:59:27.089763764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T20:59:46.475614985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T20:59:57.09172985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:00:09.22728689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38830" level=debug ts=2026-05-18T21:00:26.526879446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:00:27.095196069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:00:36.598439918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60878" ts=2026-05-18T21:00:39.228649255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49562" ts=2026-05-18T21:00:47.397449497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51362" ts=2026-05-18T21:00:57.098005831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:01:17.400262954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37646" level=debug ts=2026-05-18T21:01:22.947151709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:01:27.09959478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:01:57.101012065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:02:17.405362722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33164" level=debug ts=2026-05-18T21:02:22.829437671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:02:22.830890993Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:02:27.103669821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:02:36.610353243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49250" ts=2026-05-18T21:02:47.407997639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49332" ts=2026-05-18T21:02:57.10646833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:03:24.250536304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:03:27.108567932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=error ts=2026-05-18T21:03:27.770584564Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:03:27.772284857Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:03:27.773972309Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:03:27.773994712Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:03:39.245014565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42162" ts=2026-05-18T21:03:57.110700728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:04:09.246098688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48142" ts=2026-05-18T21:04:17.416433191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40520" level=debug ts=2026-05-18T21:04:24.781878333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:04:27.113591343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:04:39.247067528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44926" ts=2026-05-18T21:04:57.11656964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:05:07.899627835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:05:09.248376669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48550" ts=2026-05-18T21:05:17.422403704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60448" ts=2026-05-18T21:05:27.118640512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:05:57.121369982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:06:06.684293573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:06:17.428053318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56976" ts=2026-05-18T21:06:27.123902009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:06:57.1263429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:07:04.467597392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:07:04.46914586Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:07:06.634785447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50876" ts=2026-05-18T21:07:27.128791178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:07:47.436016266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60764" ts=2026-05-18T21:07:57.131660912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:08:02.052580614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:08:27.134515202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:08:36.644599314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55522" ts=2026-05-18T21:08:39.269243525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36922" level=debug ts=2026-05-18T21:08:39.397061994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:08:57.137995148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:09:27.141144731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:09:30.165801971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:09:36.649954136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57620" ts=2026-05-18T21:09:57.143875595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:10:17.449894911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49478" ts=2026-05-18T21:10:27.147096588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:10:31.452769142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:10:57.149068007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:11:09.281537095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60478" ts=2026-05-18T21:11:27.150492449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:11:28.754980561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:11:28.756567612Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:11:57.152792419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:12:04.232363296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:12:27.155708271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:12:42.102361718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:12:57.157781449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:13:06.673373326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33014" ts=2026-05-18T21:13:17.466234418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60974" ts=2026-05-18T21:13:27.159746198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=error ts=2026-05-18T21:13:27.771650305Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:13:27.773596142Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:13:27.775241894Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:13:27.775264319Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-18T21:13:31.319502079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:13:57.161731003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:14:06.680565695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33028" level=debug ts=2026-05-18T21:14:25.492075972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:14:27.163569973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:14:36.68402942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58538" ts=2026-05-18T21:14:57.166994974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:15:15.530714873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:15:15.532836525Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:15:17.477258656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59420" ts=2026-05-18T21:15:27.168829664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:15:36.690910558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:32872" ts=2026-05-18T21:15:47.480456083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59528" ts=2026-05-18T21:15:57.170901835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:16:01.614958964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:16:09.313577136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47824" ts=2026-05-18T21:16:17.483680916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56638" ts=2026-05-18T21:16:27.173649967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:16:36.697368188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56282" ts=2026-05-18T21:16:47.486689369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52046" ts=2026-05-18T21:16:57.176216287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:16:59.835790656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:17:06.700855938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51188" ts=2026-05-18T21:17:17.489917187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49214" ts=2026-05-18T21:17:27.177472447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:17:45.688545421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:17:47.492905024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34050" ts=2026-05-18T21:17:57.17999794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:18:06.707935701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36256" ts=2026-05-18T21:18:09.32282259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50304" ts=2026-05-18T21:18:27.182046514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:18:29.649933799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:18:57.183841835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:19:17.499306499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48360" level=debug ts=2026-05-18T21:19:25.042287442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:19:25.043894139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:19:27.186785841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:19:36.718464714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46016" ts=2026-05-18T21:19:57.189221957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:20:06.721906392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41470" level=debug ts=2026-05-18T21:20:12.187030674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:20:27.191682725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:20:57.194347559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:21:02.34136754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:21:09.340591495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36468" ts=2026-05-18T21:21:17.511035655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57564" ts=2026-05-18T21:21:27.197242696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:21:39.205604055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:21:57.200779967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:22:27.204288117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:22:39.847903037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:22:57.207690161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:23:24.684800815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:23:24.686366727Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:23:27.210370194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=error ts=2026-05-18T21:23:27.770627754Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:23:27.772098692Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:23:27.77355346Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:23:27.773575527Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:23:39.352945216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34166" ts=2026-05-18T21:23:57.212436548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:24:01.00322081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:24:09.354759628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43176" ts=2026-05-18T21:24:17.527268742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55444" ts=2026-05-18T21:24:27.214261442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:24:36.74576565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60244" ts=2026-05-18T21:24:39.356269081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42844" ts=2026-05-18T21:24:47.529660916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36516" level=debug ts=2026-05-18T21:24:48.717457738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:24:57.215779197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:25:17.532573773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56968" ts=2026-05-18T21:25:27.216791624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:25:45.343046304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:25:57.218415048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:26:09.363785376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47446" level=debug ts=2026-05-18T21:26:21.980431478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:26:27.220107178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:26:57.221868119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:26:57.357535157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:26:57.359101104Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:27:27.223108643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:27:53.792924828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:27:57.225430131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:28:06.764630233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49498" ts=2026-05-18T21:28:09.37472913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34908" ts=2026-05-18T21:28:27.228373554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:28:33.28635877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:28:57.231356568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:29:09.379514414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:32926" level=debug ts=2026-05-18T21:29:13.413728188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:29:27.234445843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:29:36.771923061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48324" ts=2026-05-18T21:29:57.237570151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:30:09.384919996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:52952" level=debug ts=2026-05-18T21:30:10.078478136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:30:27.240261464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:30:39.385974592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:41600" ts=2026-05-18T21:30:57.242314849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:30:58.907093186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:30:58.90856Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:31:17.565390034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45184" ts=2026-05-18T21:31:27.245775287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:31:47.567695232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42478" level=debug ts=2026-05-18T21:31:47.727317518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:31:57.24814368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:32:06.78636361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40086" ts=2026-05-18T21:32:27.25097882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:32:39.396198529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35580" level=debug ts=2026-05-18T21:32:47.786633825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:32:57.2539269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:33:23.26155364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:33:27.25742524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=error ts=2026-05-18T21:33:27.770475107Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:33:27.772039415Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:33:27.7735326Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:33:27.773556454Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:33:36.79458251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39324" ts=2026-05-18T21:33:57.259937653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:34:07.724931011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:34:27.26194415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:34:46.030942093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:34:46.032498318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:34:57.265064993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:35:06.803689493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49148" ts=2026-05-18T21:35:17.587367055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58416" level=debug ts=2026-05-18T21:35:21.290914072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:35:27.268013463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:35:36.807131349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60858" ts=2026-05-18T21:35:47.58984896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57830" level=debug ts=2026-05-18T21:35:56.230953041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:35:57.271121924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:36:06.810707918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:58374" ts=2026-05-18T21:36:09.415706108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40596" ts=2026-05-18T21:36:17.592470744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48658" ts=2026-05-18T21:36:27.274198162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:36:37.57020346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:36:47.595133896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40820" ts=2026-05-18T21:36:57.276650601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:37:06.817248854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34630" level=debug ts=2026-05-18T21:37:12.595889516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:37:27.279275492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:37:36.820093345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57414" ts=2026-05-18T21:37:39.423318202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56092" level=debug ts=2026-05-18T21:37:56.783495801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:37:56.784949237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:37:57.282551227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:38:06.823266755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34088" ts=2026-05-18T21:38:27.284972276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:38:33.711364377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:38:47.605301746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37420" ts=2026-05-18T21:38:57.285947077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:39:27.286926902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:39:32.186820627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:39:47.610681749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47034" ts=2026-05-18T21:39:57.289248233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:40:06.833585298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35202" ts=2026-05-18T21:40:09.437019863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45716" ts=2026-05-18T21:40:27.292847252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:40:27.440606597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:40:36.836278973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49394" ts=2026-05-18T21:40:39.438588393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36312" ts=2026-05-18T21:40:57.295426944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:41:10.480921315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:41:17.618556788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47044" ts=2026-05-18T21:41:27.297957544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:41:47.621934094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45830" ts=2026-05-18T21:41:57.299021092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:42:09.446697906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49798" level=debug ts=2026-05-18T21:42:11.533022891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:42:11.534701868Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:42:27.300365884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:42:47.626976987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42670" ts=2026-05-18T21:42:57.302411375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:43:08.793468758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:43:17.630081156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35590" ts=2026-05-18T21:43:27.305299302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=error ts=2026-05-18T21:43:27.771441817Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:43:27.773188912Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:43:27.774956319Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:43:27.774976213Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:43:36.855010374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60490" level=debug ts=2026-05-18T21:43:45.770688171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:43:47.632885704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54668" ts=2026-05-18T21:43:57.30801384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:44:09.457452292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48998" ts=2026-05-18T21:44:27.3105902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:44:33.200401951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:44:47.638460526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54744" ts=2026-05-18T21:44:57.313109254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:45:09.461043699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50940" level=debug ts=2026-05-18T21:45:25.958494145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:45:27.314937994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:45:39.46203836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:53942" ts=2026-05-18T21:45:47.64466161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35760" ts=2026-05-18T21:45:57.318025583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:46:09.463632304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42144" ts=2026-05-18T21:46:27.321432401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:46:27.430949738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:46:27.432447352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:46:57.32439631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:47:17.653100856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44876" level=debug ts=2026-05-18T21:47:25.424549341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:47:27.32719196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:47:39.470038603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60584" ts=2026-05-18T21:47:47.656278666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58460" ts=2026-05-18T21:47:57.330138529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:47:59.991064084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:48:06.879222692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60666" ts=2026-05-18T21:48:27.332569332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:48:36.881823986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37766" level=debug ts=2026-05-18T21:48:55.485972614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:48:57.33544788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:49:09.477553699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33066" ts=2026-05-18T21:49:27.33840895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=debug ts=2026-05-18T21:49:31.207982523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:49:36.887444744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57772" ts=2026-05-18T21:49:39.478613748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34568" ts=2026-05-18T21:49:57.340519767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:50:06.890626892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37618" ts=2026-05-18T21:50:27.342906632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:50:28.987102025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:50:28.988935439Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:50:57.346182386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:51:06.896550963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57538" level=debug ts=2026-05-18T21:51:09.183183042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:51:09.486044583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60952" ts=2026-05-18T21:51:27.348572221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:51:36.899518518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50832" ts=2026-05-18T21:51:39.487030766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50124" ts=2026-05-18T21:51:47.677724941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45710" ts=2026-05-18T21:51:57.351081996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:52:03.278146072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:52:06.902732578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44094" ts=2026-05-18T21:52:09.488567624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42604" ts=2026-05-18T21:52:27.352791763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:52:39.489500163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39988" ts=2026-05-18T21:52:47.682976547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57128" level=debug ts=2026-05-18T21:52:47.718862689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:52:57.354511744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:53:27.356472929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" level=error ts=2026-05-18T21:53:27.775711457Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:53:27.777745487Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:53:27.779510405Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=error ts=2026-05-18T21:53:27.779542873Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host" level=debug ts=2026-05-18T21:53:41.775568423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:53:57.35837964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" ts=2026-05-18T21:54:06.914350966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41478" ts=2026-05-18T21:54:17.692263074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40948" ts=2026-05-18T21:54:27.360659271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:54:35.482759265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:54:35.484275729Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:54:36.918050639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57188" ts=2026-05-18T21:54:47.695107802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44884" ts=2026-05-18T21:54:57.363877796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:55:09.503193742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39148" level=debug ts=2026-05-18T21:55:24.80364074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:55:27.367565948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:55:47.70065527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37158" ts=2026-05-18T21:55:57.370551131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:56:01.97178668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:56:27.373688428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:56:36.928986384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51654" level=debug ts=2026-05-18T21:56:41.301462733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:56:57.37715635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:57:09.51478366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55384" level=debug ts=2026-05-18T21:57:20.813083169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:57:27.379903301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" ts=2026-05-18T21:57:36.934666515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50366" ts=2026-05-18T21:57:47.711895303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49944" ts=2026-05-18T21:57:57.381862532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-lfhls-ebb7afbc 10.128.0.72:7946" level=debug ts=2026-05-18T21:58:00.808303937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" level=error ts=2026-05-18T21:58:00.809826563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host" ts=2026-05-18T21:58:09.519295784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54150" ts=2026-05-18T21:58:27.383817896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7cbad95 10.129.0.75:7946" ts=2026-05-18T21:58:36.940824651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44764" ts=2026-05-18T21:58:47.71790374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59242" ts=2026-05-18T21:58:57.386590982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-z7b5x-51bfb96d 10.130.0.68:7946" level=debug ts=2026-05-18T21:59:03.281938984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" ts=2026-05-18T21:59:06.943563527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48032"