level=info ts=2026-04-22T19:02:01.873006218Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T19:02:01.873040231Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T19:02:01.874501632Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T19:02:01.87481Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T19:02:01.874880723Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.97 inf=eth0
level=debug ts=2026-04-22T19:02:01.875273108Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T19:02:01.875329262Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.97 inf=eth0
level=info ts=2026-04-22T19:02:01.875452812Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-2ae1807a
ts=2026-04-22T19:02:01.875531035Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T19:02:01.875555671Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.97 advertisePort=7946
level=debug ts=2026-04-22T19:02:01.875574132Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.97 advertisePort=7946
level=info ts=2026-04-22T19:02:01.875724637Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T19:02:01.882901089Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:02:01.882919655Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T19:02:01.882932281Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.210194ms
level=error ts=2026-04-22T19:02:01.882941395Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T19:02:01.882955608Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T19:02:01.885447813Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:02:01.885461543Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T19:02:01.889460404Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T19:02:01.889460092Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T19:02:01.889487009Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T19:02:01.889504123Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T19:02:01.889510125Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-22T19:02:01.889540703Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T19:02:01.889598341Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-22T19:02:01.889603367Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T19:02:01.889784335Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-22T19:02:01.889794283Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T19:02:01.889834215Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T19:02:01.8898642Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T19:02:01.889898251Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T19:02:01.890054791Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T19:02:01.890064619Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T19:02:01.890075155Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T19:02:02.89072439Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-04-22T19:02:03.014605991Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:02:03.014627944Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T19:02:03.049705705Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T19:02:03.049743322Z caller=loki.go:599 msg="Loki started" startup_time=1.239439915s
ts=2026-04-22T19:02:05.350378873Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:02:05.350403378Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T19:02:08.050765857Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T19:02:08.050818551Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T19:02:11.238442274Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T19:02:11.238464763Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:02:20.855571338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38216"
ts=2026-04-22T19:02:22.563777861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51390"
ts=2026-04-22T19:02:22.592642454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.81:7946"
ts=2026-04-22T19:02:22.595256829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.79:7946"
ts=2026-04-22T19:02:22.598529332Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.82:7946: dial tcp 10.130.0.82:7946: connect: connection refused"
ts=2026-04-22T19:02:22.600196648Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
ts=2026-04-22T19:02:22.600311076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.97:7946"
ts=2026-04-22T19:02:22.60032793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.97:57090"
ts=2026-04-22T19:02:22.601768033Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.96:7946: dial tcp 10.128.0.96:7946: connect: connection refused"
level=info ts=2026-04-22T19:02:22.601780484Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=20.718818393s
level=debug ts=2026-04-22T19:02:22.641301902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:02:22.776149458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39492"
ts=2026-04-22T19:02:34.609225374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:47488"
level=debug ts=2026-04-22T19:02:41.372983104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:02:56.713310385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38998"
ts=2026-04-22T19:02:57.91227461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:03:02.653347898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T19:03:24.962935768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:03:26.716497577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41878"
ts=2026-04-22T19:03:27.915673819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:03:34.614180963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38346"
level=debug ts=2026-04-22T19:03:56.800964287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:03:56.802560452Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:03:57.919564239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:04:20.56207773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51274"
ts=2026-04-22T19:04:27.923442496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:04:42.591128772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:04:57.926968934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:05:20.566549849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56488"
ts=2026-04-22T19:05:27.93072428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:05:44.503573891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:05:56.727487064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55164"
ts=2026-04-22T19:05:57.933769818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:06:26.73075995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36330"
ts=2026-04-22T19:06:27.936823143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:06:37.081490356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:06:57.940724864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:07:04.625508321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56234"
ts=2026-04-22T19:07:27.944622473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:07:56.737058105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40622"
ts=2026-04-22T19:07:57.947784216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:08:00.063087651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:08:04.62983759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54350"
ts=2026-04-22T19:08:27.951530029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:08:50.794500765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:08:50.795964706Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:08:57.954536781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:09:04.636001365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:58498"
ts=2026-04-22T19:09:26.743750539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52872"
ts=2026-04-22T19:09:27.958284661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:09:34.639424757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48308"
level=debug ts=2026-04-22T19:09:38.854303105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:09:50.581605618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40158"
ts=2026-04-22T19:09:57.961422057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:10:20.585036757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43104"
ts=2026-04-22T19:10:27.964928157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:10:32.399285167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:10:34.644832867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33284"
ts=2026-04-22T19:10:57.967983784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:11:04.648556508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56680"
level=debug ts=2026-04-22T19:11:17.243518911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:11:27.971588158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:11:57.97483292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:11:58.929704168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:04.653296829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43590"
level=info ts=2026-04-22T19:12:08.051210918Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T19:12:08.05347437Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:12:08.055104Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:12:08.05670602Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:12:08.056737093Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:12:08.056750673Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T19:12:27.977906334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:12:34.656917392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53578"
level=debug ts=2026-04-22T19:12:39.973606158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:12:39.975405903Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:12:56.756771839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38822"
ts=2026-04-22T19:12:57.981124804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:13:27.984045502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:13:31.287012973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:13:34.661945053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56670"
ts=2026-04-22T19:13:57.987721898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:14:11.371376729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:14:26.762756499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35520"
ts=2026-04-22T19:14:27.991466346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:14:57.99537044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:15:04.246270535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:15:26.767714303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49052"
ts=2026-04-22T19:15:27.999263226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:15:50.209114271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:15:56.770873518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40318"
ts=2026-04-22T19:15:58.002388443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:16:04.670092399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:59388"
ts=2026-04-22T19:16:28.00624078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:16:42.185021647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:16:42.186767373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:16:50.605685817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35794"
ts=2026-04-22T19:16:58.010026209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:17:04.674901485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33430"
ts=2026-04-22T19:17:20.60877239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:34286"
level=debug ts=2026-04-22T19:17:27.183345265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:17:28.013644076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:17:56.778835005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51692"
ts=2026-04-22T19:17:58.016865552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:18:22.935951672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:18:26.782355971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44646"
ts=2026-04-22T19:18:28.020472979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:18:34.6811241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33448"
ts=2026-04-22T19:18:56.785741108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53698"
ts=2026-04-22T19:18:58.023910423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:19:19.741701226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:19:26.789382258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60454"
ts=2026-04-22T19:19:28.027829439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:19:34.685731919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54242"
ts=2026-04-22T19:19:56.792631745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40234"
ts=2026-04-22T19:19:58.031681127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:20:16.278777942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:20:28.034672085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:20:54.044075829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:20:54.045885709Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:20:56.797189336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47572"
ts=2026-04-22T19:20:58.038188098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:21:04.691389448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:46992"
ts=2026-04-22T19:21:28.041328807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:21:52.528451518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:21:58.044615295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=error ts=2026-04-22T19:22:08.061135562Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:22:08.063149667Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:22:08.064943362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:22:08.064972543Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:22:20.624380132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39518"
ts=2026-04-22T19:22:26.802351515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36970"
ts=2026-04-22T19:22:28.048005953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:22:45.123814314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:22:50.627983024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:42850"
ts=2026-04-22T19:22:58.051688441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:23:04.699873221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36636"
level=debug ts=2026-04-22T19:23:23.290634886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:23:28.055140477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:23:34.702842408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34158"
ts=2026-04-22T19:23:58.058623806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:24:25.196441026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:24:28.062363595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:24:58.06576568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:25:26.363325138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:25:26.364901806Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:25:28.069664069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:25:34.710767885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34534"
ts=2026-04-22T19:25:58.073236125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:26:04.714221526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37062"
ts=2026-04-22T19:26:20.638473321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:33472"
level=debug ts=2026-04-22T19:26:25.60079279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:26:28.076215498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:26:58.07982954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:27:19.697044471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:27:20.643737714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45408"
ts=2026-04-22T19:27:26.81763222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46332"
ts=2026-04-22T19:27:28.083541115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:27:56.820694959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44274"
ts=2026-04-22T19:27:58.086866021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:28:15.52511187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:28:28.090392699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:28:50.64957972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45190"
ts=2026-04-22T19:28:56.825145494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39922"
ts=2026-04-22T19:28:58.093646083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:29:04.723489829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44074"
level=debug ts=2026-04-22T19:29:09.524580463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:29:28.097343652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:29:44.700667455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:29:44.702473426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:29:58.100491925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:30:20.655192392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:32776"
ts=2026-04-22T19:30:28.103868349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:30:45.710104515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:30:56.832244525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46438"
ts=2026-04-22T19:30:58.107392388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:31:04.730939484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45974"
level=debug ts=2026-04-22T19:31:24.087832804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:31:28.111400507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:31:34.734512187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39056"
ts=2026-04-22T19:31:50.660665631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45418"
ts=2026-04-22T19:31:58.115081648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:32:04.737936698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55534"
level=error ts=2026-04-22T19:32:08.061911471Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:32:08.063665644Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:32:08.065298178Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:32:08.065330715Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T19:32:20.307084164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:32:20.664384351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:60442"
ts=2026-04-22T19:32:28.118241978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:32:50.667725437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:37494"
ts=2026-04-22T19:32:58.12214016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:33:16.128112407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:33:20.671871192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43930"
ts=2026-04-22T19:33:28.126065795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:33:50.603158177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:33:50.604749019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:33:50.675765234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39226"
ts=2026-04-22T19:33:58.129835283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:34:04.745653337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49518"
ts=2026-04-22T19:34:20.679534839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50600"
ts=2026-04-22T19:34:28.133059307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:34:34.749736854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50554"
level=debug ts=2026-04-22T19:34:36.466920916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:34:58.136909132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:35:24.360139599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:35:28.140689282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:35:56.848366852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42246"
ts=2026-04-22T19:35:58.144685727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:36:04.76338991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53172"
level=debug ts=2026-04-22T19:36:08.691689596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:36:28.148798604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:36:58.15208298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:37:04.770559845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:56398"
level=debug ts=2026-04-22T19:37:08.845710107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:37:28.155467557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:37:34.77399701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42178"
level=debug ts=2026-04-22T19:37:47.774536634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:37:47.776455714Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:37:50.691687438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40568"
ts=2026-04-22T19:37:58.160816216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:38:04.778075777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45650"
ts=2026-04-22T19:38:26.86109923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34898"
ts=2026-04-22T19:38:28.164397854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:38:41.320921849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:38:56.865124948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50566"
ts=2026-04-22T19:38:58.168359144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:39:28.171449942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:39:32.921750592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:39:50.699858812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40874"
ts=2026-04-22T19:39:56.877102688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54784"
ts=2026-04-22T19:39:58.174930654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:40:04.78591354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36676"
ts=2026-04-22T19:40:28.178662039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:40:31.605283592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:40:50.70437246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50772"
ts=2026-04-22T19:40:56.883468242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46272"
ts=2026-04-22T19:40:58.182628867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:41:11.528054514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:41:26.886589518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60138"
ts=2026-04-22T19:41:28.186675472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:41:34.793061225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55644"
ts=2026-04-22T19:41:50.70945509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51588"
ts=2026-04-22T19:41:58.190325605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=error ts=2026-04-22T19:42:08.066324903Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:42:08.069150298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:42:08.07133865Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:42:08.071362387Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T19:42:11.667737519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:42:11.669637832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:42:20.713836459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52014"
ts=2026-04-22T19:42:26.891248702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57788"
ts=2026-04-22T19:42:28.193543247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:42:52.166939304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:42:58.196674971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:43:04.799923121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44176"
ts=2026-04-22T19:43:28.200743815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:43:45.33294627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:43:56.89869536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48874"
ts=2026-04-22T19:43:58.204745826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:44:28.208649007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:44:28.647836808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:44:34.806629974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:32770"
ts=2026-04-22T19:44:50.722052194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52490"
ts=2026-04-22T19:44:58.212511024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:45:05.013724574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:45:28.216175793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:45:34.812620868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35940"
ts=2026-04-22T19:45:50.726627015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:47842"
level=debug ts=2026-04-22T19:45:57.355247289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:45:57.356866237Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:45:58.21970805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
ts=2026-04-22T19:46:28.222952298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T19:46:38.87296429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:46:58.226674072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:47:28.230591372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:47:33.599277201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:47:34.82108374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34514"
ts=2026-04-22T19:47:50.735438627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:57046"
ts=2026-04-22T19:47:56.914633552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47254"
ts=2026-04-22T19:47:58.234135534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:48:04.824794581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53944"
level=debug ts=2026-04-22T19:48:08.506012913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:26.918704317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53096"
ts=2026-04-22T19:48:28.237716906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2fb7d9a6 10.130.0.84:7946"
level=debug ts=2026-04-22T19:48:47.540232237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:50.739422709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45608"
ts=2026-04-22T19:48:58.241294117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:49:07.58825121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46018"
ts=2026-04-22T19:49:07.597084261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46022"
ts=2026-04-22T19:49:23.876868351Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2fb7d9a6 (timeout reached)"
ts=2026-04-22T19:49:26.876504728Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2fb7d9a6 has failed, no acks received"
ts=2026-04-22T19:49:28.244816746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:49:31.396268063Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-2fb7d9a6 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T19:49:48.288831428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:49:48.291292447Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:49:56.925165934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35710"
ts=2026-04-22T19:49:58.248571724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:50:04.126302748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:55448"
ts=2026-04-22T19:50:28.252974608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ad833 10.130.0.84:7946"
level=debug ts=2026-04-22T19:50:30.666069301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:50:34.1303721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:55426"
ts=2026-04-22T19:50:58.256706546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:51:04.133436865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45226"
ts=2026-04-22T19:51:04.835076446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33366"
level=debug ts=2026-04-22T19:51:24.099630898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:51:28.259862539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:51:34.838654512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:36582"
ts=2026-04-22T19:51:58.263346717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ad833 10.130.0.84:7946"
level=error ts=2026-04-22T19:52:08.064408803Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:52:08.066931443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:52:08.069278015Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:52:08.069314974Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T19:52:11.372368796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:52:26.936927255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35292"
ts=2026-04-22T19:52:28.267357793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ad833 10.130.0.84:7946"
level=debug ts=2026-04-22T19:52:45.778500657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:52:58.271300363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ad833 10.130.0.84:7946"
ts=2026-04-22T19:53:04.141748188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:44358"
ts=2026-04-22T19:53:26.942221095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:50886"
ts=2026-04-22T19:53:28.274800883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:53:35.333146964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:53:35.335135017Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:53:58.278483048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:54:04.84712603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55224"
level=debug ts=2026-04-22T19:54:09.884180589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:28.281479718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:54:50.101010234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:56.947612716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44912"
ts=2026-04-22T19:54:58.284633546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:55:04.149669793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46422"
ts=2026-04-22T19:55:26.951443759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43034"
ts=2026-04-22T19:55:28.28816933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:55:34.153072008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:34150"
ts=2026-04-22T19:55:34.853944807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38686"
level=debug ts=2026-04-22T19:55:45.083166724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:55:58.291830571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-944ad833 10.130.0.84:7946"
level=debug ts=2026-04-22T19:56:20.732538115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:56:28.295003828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:56:28.876484997Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-944ad833 (timeout reached)"
ts=2026-04-22T19:56:31.875987471Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-944ad833 has failed, no acks received"
ts=2026-04-22T19:56:34.859088872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57698"
ts=2026-04-22T19:56:37.211475489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54718"
ts=2026-04-22T19:56:37.22002151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54722"
ts=2026-04-22T19:56:43.877330356Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-944ad833 (timeout reached)"
ts=2026-04-22T19:56:46.399258905Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-944ad833 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:56:46.876660625Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-944ad833 has failed, no acks received"
ts=2026-04-22T19:56:56.958782058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43308"
ts=2026-04-22T19:56:58.298151052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T19:57:04.862475625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38272"
level=debug ts=2026-04-22T19:57:12.221299564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:57:12.223357182Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:57:28.302189705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58f2dd7c 10.130.0.84:7946"
ts=2026-04-22T19:57:34.866305165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55300"
level=debug ts=2026-04-22T19:57:56.348237268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:57:56.68629238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51560"
ts=2026-04-22T19:57:56.966692697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55692"
ts=2026-04-22T19:57:58.305891621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T19:58:26.689879826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:57330"
ts=2026-04-22T19:58:28.309643226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:58:34.004309722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:58:56.973597447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57454"
ts=2026-04-22T19:58:58.313006459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T19:59:11.066564346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:59:26.977576433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48700"
ts=2026-04-22T19:59:28.316858181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58f2dd7c 10.130.0.84:7946"
level=debug ts=2026-04-22T19:59:52.687796825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:59:58.320714641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58f2dd7c 10.130.0.84:7946"
ts=2026-04-22T20:00:26.696625177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40566"
ts=2026-04-22T20:00:28.324098444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58f2dd7c 10.130.0.84:7946"
level=debug ts=2026-04-22T20:00:34.212678405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:00:34.215274515Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:00:34.876745319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:39082"
ts=2026-04-22T20:00:56.700108845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56474"
ts=2026-04-22T20:00:58.328269963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:01:19.547163674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:01:28.332170961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:01:56.986571298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58080"
ts=2026-04-22T20:01:58.336100653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:02:03.876792793Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-58f2dd7c (timeout reached)"
ts=2026-04-22T20:02:04.883097939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48272"
ts=2026-04-22T20:02:06.876295277Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-58f2dd7c has failed, no acks received"
level=error ts=2026-04-22T20:02:08.062746625Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:02:08.065106411Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:02:08.0672824Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:02:08.067307176Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:02:18.876791527Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-58f2dd7c (timeout reached)"
level=debug ts=2026-04-22T20:02:18.919297666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:02:21.876216541Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-58f2dd7c has failed, no acks received"
ts=2026-04-22T20:02:26.396714743Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-58f2dd7c as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:02:26.990156035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60242"
ts=2026-04-22T20:02:28.339369513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:02:31.380173042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35248"
ts=2026-04-22T20:02:31.38856307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35252"
ts=2026-04-22T20:02:56.993488956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54616"
level=debug ts=2026-04-22T20:02:57.261668618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:02:58.342695944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:03:20.777056783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:42650"
ts=2026-04-22T20:03:26.997165017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54406"
ts=2026-04-22T20:03:28.346487878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82eeb516 10.130.0.84:7946"
ts=2026-04-22T20:03:34.889831146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43628"
level=debug ts=2026-04-22T20:03:46.881459554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:03:50.780631251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56246"
ts=2026-04-22T20:03:58.350241889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:04:21.497739041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:04:21.499637111Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:04:28.353966784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82eeb516 10.130.0.84:7946"
ts=2026-04-22T20:04:50.785130034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50698"
ts=2026-04-22T20:04:57.003539417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45840"
ts=2026-04-22T20:04:58.357878473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:05:04.927541582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:05:28.362093538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:05:50.789544012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:53992"
ts=2026-04-22T20:05:58.365545865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:06:02.829177956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:06:04.898809588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:45566"
ts=2026-04-22T20:06:27.009282886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38624"
ts=2026-04-22T20:06:28.368567746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:06:34.902234071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42434"
ts=2026-04-22T20:06:50.794386181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:56078"
level=debug ts=2026-04-22T20:06:54.951410235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:06:58.372413658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:07:04.906127538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37558"
ts=2026-04-22T20:07:27.014032879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55502"
ts=2026-04-22T20:07:28.376934853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:07:34.909681239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:46740"
level=debug ts=2026-04-22T20:07:37.424526018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:07:50.799233309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50358"
ts=2026-04-22T20:07:57.017403246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38504"
ts=2026-04-22T20:07:58.380836377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82eeb516 10.130.0.84:7946"
ts=2026-04-22T20:08:28.384424302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-82eeb516 10.130.0.84:7946"
level=debug ts=2026-04-22T20:08:32.148781597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:08:32.150477992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:08:58.388087735Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-82eeb516 failed: dial tcp 10.130.0.84:7946: connect: connection refused"
ts=2026-04-22T20:08:58.877461233Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-82eeb516 (timeout reached)"
ts=2026-04-22T20:09:01.876768691Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-82eeb516 has failed, no acks received"
ts=2026-04-22T20:09:04.916368459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52820"
ts=2026-04-22T20:09:13.87706285Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-82eeb516 (timeout reached)"
ts=2026-04-22T20:09:16.876387717Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-82eeb516 has failed, no acks received"
ts=2026-04-22T20:09:21.877180816Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-82eeb516 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:09:23.876263782Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-82eeb516 (timeout reached)"
ts=2026-04-22T20:09:26.876118914Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-82eeb516 has failed, no acks received"
level=debug ts=2026-04-22T20:09:27.293184567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:09:28.390541068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:09:46.175637121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50104"
ts=2026-04-22T20:09:46.184813045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50112"
ts=2026-04-22T20:09:58.394110354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a10359d1 10.130.0.84:7946"
ts=2026-04-22T20:10:04.921348863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:35238"
level=debug ts=2026-04-22T20:10:10.923490831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:10:28.397069738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:10:38.338547871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:41072"
level=debug ts=2026-04-22T20:10:46.621457383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:10:58.400787941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a10359d1 10.130.0.84:7946"
ts=2026-04-22T20:11:27.028600127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39840"
ts=2026-04-22T20:11:28.404182404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:11:42.578511165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:11:58.408138243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=error ts=2026-04-22T20:12:08.060654825Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:12:08.062549209Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:12:08.064351885Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:12:08.064374955Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:12:27.033382705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60690"
ts=2026-04-22T20:12:28.411658073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:12:40.121572469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:12:40.123321962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:12:58.414641869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:13:23.988048199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:13:28.417563152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a10359d1 10.130.0.84:7946"
ts=2026-04-22T20:13:58.420783903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:14:19.630957496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:14:27.040295483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46720"
ts=2026-04-22T20:14:28.424565125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:14:38.351068581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:45514"
ts=2026-04-22T20:14:58.428040299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a10359d1 10.130.0.84:7946"
ts=2026-04-22T20:15:04.939039721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48218"
ts=2026-04-22T20:15:08.354215148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52396"
level=debug ts=2026-04-22T20:15:15.419725678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:15:28.431595261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:15:34.942011574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41512"
ts=2026-04-22T20:15:58.43514851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:16:04.945831936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:37126"
ts=2026-04-22T20:16:08.358821441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52464"
level=debug ts=2026-04-22T20:16:08.647834152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:16:13.877091458Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a10359d1 (timeout reached)"
ts=2026-04-22T20:16:16.876676445Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a10359d1 has failed, no acks received"
ts=2026-04-22T20:16:23.87619036Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a10359d1 (timeout reached)"
ts=2026-04-22T20:16:26.876586713Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a10359d1 has failed, no acks received"
ts=2026-04-22T20:16:28.438722191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:16:36.877088524Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-a10359d1 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:16:38.876473239Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a10359d1 (timeout reached)"
ts=2026-04-22T20:16:41.876547904Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a10359d1 has failed, no acks received"
level=debug ts=2026-04-22T20:16:55.160270304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:16:55.162257951Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:16:58.442270997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:17:27.050932569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52562"
ts=2026-04-22T20:17:28.446194529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:17:30.242539481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:17:33.165714096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51136"
ts=2026-04-22T20:17:33.175239361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:51148"
ts=2026-04-22T20:17:58.450213426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3456817 10.130.0.84:7946"
level=debug ts=2026-04-22T20:18:16.009343758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:27.055644258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56868"
ts=2026-04-22T20:18:28.453703377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:18:57.0594676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38668"
ts=2026-04-22T20:18:58.457151567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:19:04.162717561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:19:28.461117744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:19:58.464698814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:19:59.729458713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:20:28.468044262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:20:42.387804513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:20:42.389370721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:20:55.612913885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:53142"
ts=2026-04-22T20:20:58.471265425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3456817 10.130.0.84:7946"
ts=2026-04-22T20:21:27.068366825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38450"
ts=2026-04-22T20:21:28.474791798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:21:34.281159972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:21:55.618252898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:47908"
ts=2026-04-22T20:21:58.478534251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=error ts=2026-04-22T20:22:08.062449517Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:22:08.064411492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:22:08.066273651Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:22:08.066310536Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:22:28.482040349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:22:28.838800938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:22:57.074082691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59336"
ts=2026-04-22T20:22:58.485368277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3456817 10.130.0.84:7946"
ts=2026-04-22T20:23:04.966379724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52744"
level=debug ts=2026-04-22T20:23:18.727498301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:23:28.489249999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3456817 10.130.0.84:7946"
ts=2026-04-22T20:23:58.492571887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3456817 10.130.0.84:7946"
level=debug ts=2026-04-22T20:24:19.745614186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:27.079825193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54764"
ts=2026-04-22T20:24:28.49566992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:24:56.669367294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:24:56.67108446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:24:58.499013441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:25:28.502197333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:25:34.974421974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:54050"
level=debug ts=2026-04-22T20:25:45.282757232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:25:58.505579273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:25:58.87659004Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d3456817 (timeout reached)"
ts=2026-04-22T20:26:01.876106077Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d3456817 has failed, no acks received"
ts=2026-04-22T20:26:11.595276059Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-d3456817 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:26:27.087282678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33954"
ts=2026-04-22T20:26:28.50872643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:26:34.979280787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33752"
level=debug ts=2026-04-22T20:26:43.508792777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:26:58.512612906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:27:27.969755635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:27:28.515942887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:27:34.983683838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:49084"
ts=2026-04-22T20:27:47.164993313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50276"
ts=2026-04-22T20:27:47.173654713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50286"
ts=2026-04-22T20:27:58.519524824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bce49153 10.130.0.84:7946"
ts=2026-04-22T20:28:27.094870496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47584"
ts=2026-04-22T20:28:28.522908628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bce49153 10.130.0.84:7946"
level=debug ts=2026-04-22T20:28:28.738311567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:28:34.989455284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41998"
ts=2026-04-22T20:28:57.09822806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33142"
ts=2026-04-22T20:28:58.526025972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:29:20.761082373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:29:20.76290362Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:29:27.102230248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55318"
ts=2026-04-22T20:29:28.529693971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bce49153 10.130.0.84:7946"
ts=2026-04-22T20:29:34.993973145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53216"
ts=2026-04-22T20:29:58.533605566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:30:17.025078948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:30:28.536976335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:30:58.540150853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:31:05.00071677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44684"
ts=2026-04-22T20:31:05.275593023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:32944"
level=debug ts=2026-04-22T20:31:07.002776842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:31:28.543687263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:31:58.547680235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:32:00.535190929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:05.005920923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55132"
level=error ts=2026-04-22T20:32:08.059475362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:32:08.061317043Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:32:08.062980168Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:32:08.063004204Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:32:27.112973126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55778"
ts=2026-04-22T20:32:28.55136612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:32:35.009169048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38372"
level=debug ts=2026-04-22T20:32:42.193359104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:58.554714139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:33:05.283959956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:59382"
ts=2026-04-22T20:33:27.117730796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51498"
ts=2026-04-22T20:33:28.558346465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bce49153 10.130.0.84:7946"
ts=2026-04-22T20:33:35.287591013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:35428"
level=debug ts=2026-04-22T20:33:42.616991112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:33:42.6186538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:33:57.12097761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39240"
ts=2026-04-22T20:33:58.561667117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-bce49153 10.130.0.84:7946"
ts=2026-04-22T20:34:05.015017313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43356"
ts=2026-04-22T20:34:18.87732522Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-bce49153 (timeout reached)"
ts=2026-04-22T20:34:21.877067823Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-bce49153 has failed, no acks received"
ts=2026-04-22T20:34:28.165271826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49570"
ts=2026-04-22T20:34:28.17346807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49572"
ts=2026-04-22T20:34:28.564805971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:34:31.664982825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:34:33.876803773Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-bce49153 (timeout reached)"
ts=2026-04-22T20:34:35.01836367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:53250"
ts=2026-04-22T20:34:36.594753306Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-bce49153 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:34:36.876450142Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-bce49153 has failed, no acks received"
ts=2026-04-22T20:34:58.56868967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:35:02.893968851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:57304"
level=debug ts=2026-04-22T20:35:24.087090089Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:35:28.572261375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3885247f 10.130.0.84:7946"
ts=2026-04-22T20:35:58.575459386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:36:02.898870499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:37504"
ts=2026-04-22T20:36:05.024247138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:43856"
level=debug ts=2026-04-22T20:36:25.534800095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:36:27.130110027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57044"
ts=2026-04-22T20:36:28.579020982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:36:35.027842363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55768"
ts=2026-04-22T20:36:57.133137428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45448"
ts=2026-04-22T20:36:58.582784875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:37:05.0317917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:52916"
level=debug ts=2026-04-22T20:37:21.39811688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:37:28.586013022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:37:32.905858496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:40516"
ts=2026-04-22T20:37:58.590084546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:37:59.762977311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:37:59.764590062Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:38:02.909819428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:59178"
ts=2026-04-22T20:38:27.139601458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59994"
ts=2026-04-22T20:38:28.593778839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:38:32.913692455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:49634"
ts=2026-04-22T20:38:35.037956067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:57200"
level=debug ts=2026-04-22T20:38:36.145902758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:38:58.597714054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3885247f 10.130.0.84:7946"
ts=2026-04-22T20:39:05.041072248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:42900"
level=debug ts=2026-04-22T20:39:20.919892355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:39:27.144258245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43606"
ts=2026-04-22T20:39:28.600866694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3885247f 10.130.0.84:7946"
ts=2026-04-22T20:39:32.918682781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52658"
ts=2026-04-22T20:39:35.044529685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:55672"
ts=2026-04-22T20:39:57.147472535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33198"
ts=2026-04-22T20:39:58.604649632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:40:02.922304971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:39024"
ts=2026-04-22T20:40:05.04766588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:50546"
level=debug ts=2026-04-22T20:40:11.321587395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:40:27.151467468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46162"
ts=2026-04-22T20:40:28.608541868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:40:45.732607651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:40:58.612114871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3885247f 10.130.0.84:7946"
ts=2026-04-22T20:41:28.615252057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:41:35.035050452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:41:35.036558245Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:41:58.619179995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:42:03.877326752Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3885247f (timeout reached)"
ts=2026-04-22T20:42:05.056525909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:41764"
ts=2026-04-22T20:42:06.876728701Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3885247f has failed, no acks received"
level=error ts=2026-04-22T20:42:08.063960643Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:42:08.066302347Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:42:08.068480424Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:42:08.068505725Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:42:18.876492283Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3885247f (timeout reached)"
ts=2026-04-22T20:42:19.175060875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52570"
ts=2026-04-22T20:42:19.184382546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:52580"
ts=2026-04-22T20:42:21.876243409Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3885247f has failed, no acks received"
level=debug ts=2026-04-22T20:42:28.458932016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:42:28.622845717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-edddef5c 10.130.0.84:7946"
ts=2026-04-22T20:42:35.060014779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:34776"
ts=2026-04-22T20:42:58.626033852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:43:16.103624131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:43:27.162229187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47836"
ts=2026-04-22T20:43:28.629835737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:43:57.165519649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33568"
ts=2026-04-22T20:43:58.633163636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:44:01.44546793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:44:28.636538999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:44:35.068822286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:60540"
level=debug ts=2026-04-22T20:44:47.372943727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:44:58.640067351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:45:28.643939811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:45:43.682852682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:45:43.684729225Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:45:57.173355474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59476"
ts=2026-04-22T20:45:58.646905978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:46:03.88774277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:48780"
ts=2026-04-22T20:46:05.075642779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48886"
ts=2026-04-22T20:46:28.650264887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:46:39.361535587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:46:58.653537194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-edddef5c 10.130.0.84:7946"
level=debug ts=2026-04-22T20:47:17.087895558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:47:27.179527358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59770"
ts=2026-04-22T20:47:28.657926873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-edddef5c 10.130.0.84:7946"
ts=2026-04-22T20:47:35.082088319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:48906"
level=debug ts=2026-04-22T20:47:53.474812448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:47:58.661353356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:48:28.664802619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:48:33.895887828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:54946"
ts=2026-04-22T20:48:35.08750499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:38158"
level=debug ts=2026-04-22T20:48:52.175383172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:48:58.668688909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:49:28.673320276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:49:33.901274623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:38346"
level=debug ts=2026-04-22T20:49:48.369861352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:49:48.371326027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:49:57.188544952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47060"
ts=2026-04-22T20:49:58.6772674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:50:27.191876699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55222"
ts=2026-04-22T20:50:28.680768336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
ts=2026-04-22T20:50:33.906155795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:37894"
level=debug ts=2026-04-22T20:50:48.29570042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:50:57.195251976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:59262"
ts=2026-04-22T20:50:58.684179039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:51:27.198511986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52828"
ts=2026-04-22T20:51:28.688155559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:51:31.723589224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:51:33.912265078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:46148"
ts=2026-04-22T20:51:58.691880104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:51:58.877088718Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-edddef5c (timeout reached)"
ts=2026-04-22T20:52:01.87663634Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-edddef5c has failed, no acks received"
ts=2026-04-22T20:52:05.098189813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:44812"
level=error ts=2026-04-22T20:52:08.063238763Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:52:08.065301546Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:52:08.067269589Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:52:08.067312626Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:52:18.876831679Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-edddef5c (timeout reached)"
ts=2026-04-22T20:52:21.594807314Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-edddef5c as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:52:21.876802645Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-edddef5c has failed, no acks received"
ts=2026-04-22T20:52:22.169063159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43414"
ts=2026-04-22T20:52:22.177567041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:43422"
ts=2026-04-22T20:52:28.694987115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:52:32.89961076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:52:57.204381078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39838"
ts=2026-04-22T20:52:58.698191036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:53:05.104027759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.81:33066"
level=debug ts=2026-04-22T20:53:26.657034561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:53:28.701628014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2125d014 10.130.0.84:7946"
ts=2026-04-22T20:53:58.705571993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:54:12.040398636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:54:12.042117444Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:54:28.709288662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
ts=2026-04-22T20:54:34.759278645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:50302"
ts=2026-04-22T20:54:57.211783776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44088"
ts=2026-04-22T20:54:58.71252976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:55:07.108705285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:55:28.715649008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2125d014 10.130.0.84:7946"
ts=2026-04-22T20:55:57.217357598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47526"
ts=2026-04-22T20:55:58.719268843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hhdz7-3aefa32f 10.130.0.79:7946"
level=debug ts=2026-04-22T20:55:59.746872094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:56:04.765386387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:32960"
ts=2026-04-22T20:56:28.722804678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2125d014 10.130.0.84:7946"
level=debug ts=2026-04-22T20:56:36.483618642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:56:58.726304458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2125d014 10.130.0.84:7946"
ts=2026-04-22T20:57:27.223197889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44040"
ts=2026-04-22T20:57:28.729732576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qd4qt-5c75b587 10.130.0.81:7946"
level=debug ts=2026-04-22T20:57:30.629357024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:57:34.771857417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.84:58672"