level=info ts=2026-04-23T11:38:28.70214883Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-23T11:38:28.702196654Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-23T11:38:28.703768262Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-23T11:38:28.704033839Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T11:38:28.704129373Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.96 inf=eth0
level=debug ts=2026-04-23T11:38:28.704501559Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-23T11:38:28.704556129Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.96 inf=eth0
level=info ts=2026-04-23T11:38:28.704591321Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b37ae0a1
ts=2026-04-23T11:38:28.704653549Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-23T11:38:28.704665795Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.96 advertisePort=7946
level=debug ts=2026-04-23T11:38:28.704679747Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.96 advertisePort=7946
level=info ts=2026-04-23T11:38:28.704780775Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-23T11:38:28.709501235Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T11:38:28.70951633Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-23T11:38:28.709524739Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.745793ms
level=error ts=2026-04-23T11:38:28.709531325Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-23T11:38:28.709542177Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-23T11:38:28.711645103Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T11:38:28.711663855Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T11:38:28.720600509Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-23T11:38:28.720611054Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-23T11:38:28.720616596Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-23T11:38:28.720634887Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-23T11:38:28.720653276Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-23T11:38:28.720669314Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-23T11:38:28.720730659Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-23T11:38:28.72074555Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-23T11:38:28.720848517Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-23T11:38:28.720874736Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-23T11:38:28.72095006Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-23T11:38:28.720949126Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-23T11:38:28.720976286Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-23T11:38:28.721184237Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-23T11:38:28.834787601Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-23T11:38:28.834857367Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-23T11:38:29.835840912Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-23T11:38:29.967394721Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-23T11:38:29.967454702Z caller=loki.go:599 msg="Loki started" startup_time=1.324697432s
ts=2026-04-23T11:38:30.29808644Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T11:38:30.298127554Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:38:32.616376431Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T11:38:32.616402231Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-23T11:38:34.967729245Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-23T11:38:34.967795697Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-23T11:38:37.482044056Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-23T11:38:37.482075602Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-23T11:38:48.806270005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:38:50.27294858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.70:7946"
ts=2026-04-23T11:38:50.275941439Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.69:7946: dial tcp 10.129.0.69:7946: connect: connection refused"
ts=2026-04-23T11:38:50.276989415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.94:7946"
ts=2026-04-23T11:38:50.277700651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.96:7946"
ts=2026-04-23T11:38:50.277821334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.96:45688"
level=info ts=2026-04-23T11:38:50.278152334Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.56860562s
ts=2026-04-23T11:38:51.564682146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:46800"
ts=2026-04-23T11:38:52.257420242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35864"
ts=2026-04-23T11:38:56.343510761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:46276"
ts=2026-04-23T11:39:02.641456173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:51912"
ts=2026-04-23T11:39:04.46232117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59634"
level=debug ts=2026-04-23T11:39:10.396368976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:39:11.967561654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:39:35.556269513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:39:41.969447605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T11:40:01.866178882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:40:02.628648232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:34560"
ts=2026-04-23T11:40:02.64548091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:51546"
ts=2026-04-23T11:40:11.971741604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:40:32.631586427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:51160"
ts=2026-04-23T11:40:34.468466446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38706"
level=debug ts=2026-04-23T11:40:36.543974499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T11:40:36.545701094Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:40:41.974221322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:41:02.634874621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:46574"
ts=2026-04-23T11:41:04.471573619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51158"
ts=2026-04-23T11:41:11.9763708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:41:21.031590961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:41:32.652508367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:60834"
ts=2026-04-23T11:41:41.978684786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T11:42:11.980421831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:42:14.57389254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:42:34.478914074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44650"
ts=2026-04-23T11:42:41.981334253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:43:04.480696097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52784"
ts=2026-04-23T11:43:11.983421951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T11:43:16.582953149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:43:32.650270957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:56362"
ts=2026-04-23T11:43:34.483198245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60648"
ts=2026-04-23T11:43:41.985421603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T11:44:02.653331509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37658"
ts=2026-04-23T11:44:02.664475681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:49298"
ts=2026-04-23T11:44:11.987523277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:44:15.750297594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-23T11:44:32.665415649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:35090"
ts=2026-04-23T11:44:41.990241353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:45:02.658829111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43186"
ts=2026-04-23T11:45:02.666613255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:53896"
ts=2026-04-23T11:45:11.992964026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T11:45:21.458828263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-23T11:45:21.460468856Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:45:32.668051622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:45430"
ts=2026-04-23T11:45:41.995617898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:45:56.508322614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:46:04.495297155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45284"
ts=2026-04-23T11:46:11.997528328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T11:46:35.991684242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:46:42.000494892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:47:12.003216659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:47:32.674008996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53428"
level=debug ts=2026-04-23T11:47:34.721494088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:47:42.0063453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:48:12.008618954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=info ts=2026-04-23T11:48:34.968412151Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-23T11:48:34.97063395Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:48:34.972478336Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:48:34.974131541Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:48:34.974163035Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-23T11:48:34.9741762Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-23T11:48:36.199471819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:48:42.01143492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:49:02.682205796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42210"
ts=2026-04-23T11:49:12.013582193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:49:22.677724785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T11:49:22.679749402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:49:32.684006067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:49310"
ts=2026-04-23T11:49:34.512787136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56448"
ts=2026-04-23T11:49:42.015766609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T11:50:00.010719047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:50:12.018428656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T11:50:42.019807163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:50:55.10439223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:51:02.690442416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39206"
ts=2026-04-23T11:51:04.520971786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43388"
ts=2026-04-23T11:51:12.021207301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:51:32.696393591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42450"
ts=2026-04-23T11:51:42.023062676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:51:46.089214489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:52:12.024892056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:52:24.914146975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:52:42.026893444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T11:53:00.738079101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T11:53:00.740288822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:53:12.029508244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T11:53:32.703649958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:33766"
ts=2026-04-23T11:53:42.031356299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T11:54:00.601211601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:54:02.705169963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:54358"
ts=2026-04-23T11:54:12.033496147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:54:32.70707629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:36298"
ts=2026-04-23T11:54:42.035483104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T11:55:01.385060308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:55:02.708335943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:54436"
ts=2026-04-23T11:55:12.038120752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:55:42.040034727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T11:55:51.577679312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:56:12.041443236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:56:26.671475595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:56:32.714743862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:60622"
ts=2026-04-23T11:56:32.728961531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43384"
ts=2026-04-23T11:56:42.042936447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T11:57:02.731948356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:60032"
ts=2026-04-23T11:57:12.043995952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:57:28.29877504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T11:57:28.301169801Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:57:32.734607837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35626"
ts=2026-04-23T11:57:34.556400753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47966"
ts=2026-04-23T11:57:42.045561669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T11:58:12.047370685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:58:20.869973542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:58:34.56096734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41628"
level=error ts=2026-04-23T11:58:34.978136022Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:58:34.980213936Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:58:34.98225481Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T11:58:34.982290538Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T11:58:42.049399781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T11:59:02.726049825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39134"
ts=2026-04-23T11:59:02.742724191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:57390"
ts=2026-04-23T11:59:04.563314071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57266"
ts=2026-04-23T11:59:12.051634745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T11:59:17.519455153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T11:59:32.727766222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:45578"
ts=2026-04-23T11:59:34.566290436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37136"
ts=2026-04-23T11:59:42.052994461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:00:04.570156104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54218"
level=debug ts=2026-04-23T12:00:07.89771426Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:00:12.055374035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:00:32.732752599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:50154"
ts=2026-04-23T12:00:34.572835641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48480"
ts=2026-04-23T12:00:42.058086126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T12:00:49.193786519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:01:02.734707426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:54334"
ts=2026-04-23T12:01:12.061036151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:01:32.736070891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:55668"
ts=2026-04-23T12:01:32.759329379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47816"
level=debug ts=2026-04-23T12:01:36.929147739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:01:36.930975268Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:01:42.062509603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:02:02.762234143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53750"
ts=2026-04-23T12:02:12.064964804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:02:18.097126017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:02:32.739738915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:50086"
ts=2026-04-23T12:02:32.765793118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:49052"
ts=2026-04-23T12:02:42.067246432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T12:03:02.76926011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55114"
ts=2026-04-23T12:03:04.588555655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47270"
ts=2026-04-23T12:03:12.06934369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:03:16.374772779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:03:42.072145393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T12:04:02.746277085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39808"
ts=2026-04-23T12:04:12.075070631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:04:17.990643161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:04:42.077576287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:05:04.292988425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:05:04.601546726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39432"
ts=2026-04-23T12:05:12.078887831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:05:42.081670253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
level=debug ts=2026-04-23T12:05:42.469953732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:05:42.472446966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:06:02.756194553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:45716"
ts=2026-04-23T12:06:12.085004029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9fed6b37 10.130.0.62:7946"
ts=2026-04-23T12:06:32.75718761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:33204"
ts=2026-04-23T12:06:32.799694975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41022"
ts=2026-04-23T12:06:42.087845066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:06:44.25860088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:07:02.758689284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:43880"
ts=2026-04-23T12:07:02.802993653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:57676"
ts=2026-04-23T12:07:12.089794407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:07:32.760556272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:51662"
ts=2026-04-23T12:07:42.092383106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:07:43.231863275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:08:02.809237008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42612"
ts=2026-04-23T12:08:12.093995539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:08:32.093743083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:08:34.374895706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42532"
ts=2026-04-23T12:08:34.39444801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42548"
level=error ts=2026-04-23T12:08:34.977894222Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:08:34.980423849Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:08:34.982643641Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:08:34.982676737Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:08:42.096351975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f7d7fa97 10.130.0.62:7946"
ts=2026-04-23T12:08:50.705885905Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9fed6b37 (timeout reached)"
ts=2026-04-23T12:08:53.705937708Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9fed6b37 has failed, no acks received"
ts=2026-04-23T12:08:58.700247604Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-9fed6b37 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:09:12.099429069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f7d7fa97 10.130.0.62:7946"
level=debug ts=2026-04-23T12:09:18.734001388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:09:42.102062187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:10:02.772930921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:53728"
ts=2026-04-23T12:10:12.103888816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:10:12.11695096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:10:12.118845041Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:10:26.680424545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:39154"
ts=2026-04-23T12:10:32.774914653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39664"
ts=2026-04-23T12:10:34.63256141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40032"
ts=2026-04-23T12:10:42.105615606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:10:51.316599953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:11:02.776263419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:49720"
ts=2026-04-23T12:11:12.107801872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:11:26.688143526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:50748"
ts=2026-04-23T12:11:34.637672848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35902"
level=debug ts=2026-04-23T12:11:38.566381982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:11:42.110843747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f7d7fa97 10.130.0.62:7946"
ts=2026-04-23T12:11:56.691552631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:51712"
ts=2026-04-23T12:12:04.639895942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36978"
ts=2026-04-23T12:12:12.113407777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:12:32.783163316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:59528"
level=debug ts=2026-04-23T12:12:37.740505218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:12:42.115494943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:13:04.64450014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38780"
ts=2026-04-23T12:13:12.118467765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f7d7fa97 10.130.0.62:7946"
level=debug ts=2026-04-23T12:13:20.288184565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:13:26.700670524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:45452"
ts=2026-04-23T12:13:42.121494891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:13:56.704344246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:38668"
ts=2026-04-23T12:14:02.790090371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:55144"
ts=2026-04-23T12:14:12.123328661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:14:19.84197921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:14:19.843692852Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:14:26.707363044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53040"
ts=2026-04-23T12:14:32.79183947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:59834"
ts=2026-04-23T12:14:42.125854619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:15:12.128529735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:15:16.196571446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:15:34.656234281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39784"
ts=2026-04-23T12:15:42.130448935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:16:02.79855611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:35796"
level=debug ts=2026-04-23T12:16:03.163506726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:16:04.657850209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45140"
ts=2026-04-23T12:16:12.132245974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:16:42.134683246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:17:00.197881608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:17:02.802718687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:33340"
ts=2026-04-23T12:17:12.137069395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:17:26.724223029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:49902"
ts=2026-04-23T12:17:32.804166019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:37832"
ts=2026-04-23T12:17:34.665504556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41222"
ts=2026-04-23T12:17:42.139056552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:17:48.086215342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:17:55.706435314Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f7d7fa97 (timeout reached)"
ts=2026-04-23T12:17:56.646944881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47350"
ts=2026-04-23T12:17:56.65933093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47364"
ts=2026-04-23T12:17:58.706514035Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f7d7fa97 has failed, no acks received"
ts=2026-04-23T12:18:04.66820886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44970"
ts=2026-04-23T12:18:07.099443596Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-f7d7fa97 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:18:12.141212621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=error ts=2026-04-23T12:18:34.978477402Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:18:34.980686162Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:18:34.982492918Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:18:34.982522867Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:18:42.143782829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-99c16282 10.130.0.62:7946"
level=debug ts=2026-04-23T12:18:45.74484922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:18:45.746553041Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:19:12.146677027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:19:29.722657663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:19:32.813302988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:46904"
ts=2026-04-23T12:19:42.148188804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:20:04.679076438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59628"
level=debug ts=2026-04-23T12:20:06.280980303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:20:12.150260094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:20:42.151772996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:20:52.754529218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:21:12.153274875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:21:30.013296194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:21:34.687961199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58434"
ts=2026-04-23T12:21:42.155301118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:22:04.916315197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:22:04.918069888Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:22:12.157227324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:22:25.826420958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:50024"
ts=2026-04-23T12:22:32.830373715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39056"
ts=2026-04-23T12:22:42.159202397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:22:55.829966688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41124"
level=debug ts=2026-04-23T12:23:04.520259267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:23:12.161616871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-99c16282 10.130.0.62:7946"
ts=2026-04-23T12:23:32.834352094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:52278"
ts=2026-04-23T12:23:42.163449025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:23:55.836867913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47352"
level=debug ts=2026-04-23T12:23:58.204532871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:24:02.835897541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:46468"
ts=2026-04-23T12:24:12.165862638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:24:35.853072182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:24:42.169186932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-99c16282 10.130.0.62:7946"
ts=2026-04-23T12:25:02.840442785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:33086"
level=debug ts=2026-04-23T12:25:10.635434968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:25:10.705659882Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-99c16282 (timeout reached)"
ts=2026-04-23T12:25:12.172442971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:25:13.705908665Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-99c16282 has failed, no acks received"
ts=2026-04-23T12:25:28.650581098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43204"
ts=2026-04-23T12:25:28.662657797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43210"
ts=2026-04-23T12:25:28.700040404Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-99c16282 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:25:32.841875602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:53782"
ts=2026-04-23T12:25:42.174375712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:26:02.900556276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:26:02.902320673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:26:12.178192053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:26:32.846220208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:47574"
ts=2026-04-23T12:26:42.179837083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:26:54.060923242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:27:12.181872259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:27:26.226483125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:40858"
ts=2026-04-23T12:27:34.726649809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56046"
ts=2026-04-23T12:27:42.183818847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:27:51.45740021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:28:04.729291509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42816"
ts=2026-04-23T12:28:12.185879034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-db9902e6 10.130.0.62:7946"
level=error ts=2026-04-23T12:28:34.97732623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:28:34.979199404Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:28:34.981039903Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:28:34.981064699Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-23T12:28:35.819883846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:28:42.187774507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:29:12.18918588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:29:22.421222936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:29:34.736267821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45434"
ts=2026-04-23T12:29:42.19133503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:30:04.738801792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45778"
ts=2026-04-23T12:30:12.194069341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-db9902e6 10.130.0.62:7946"
level=debug ts=2026-04-23T12:30:23.260445052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:30:23.261946848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:30:34.741052246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50712"
ts=2026-04-23T12:30:42.196551812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:31:04.743807481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39556"
level=debug ts=2026-04-23T12:31:05.161895026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:31:12.197687242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:31:42.200265427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-db9902e6 10.130.0.62:7946"
level=debug ts=2026-04-23T12:31:54.388796196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:32:04.749453087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48184"
ts=2026-04-23T12:32:12.203199994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-db9902e6 10.130.0.62:7946"
level=debug ts=2026-04-23T12:32:30.335806908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:32:30.706523473Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-db9902e6 (timeout reached)"
ts=2026-04-23T12:32:32.876070227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:57964"
ts=2026-04-23T12:32:33.706462424Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-db9902e6 has failed, no acks received"
ts=2026-04-23T12:32:34.752121917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39344"
ts=2026-04-23T12:32:40.706458784Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-db9902e6 (timeout reached)"
ts=2026-04-23T12:32:42.205045952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:32:43.705761278Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-db9902e6 has failed, no acks received"
ts=2026-04-23T12:32:48.90043253Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-db9902e6 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:33:01.655639363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:44128"
ts=2026-04-23T12:33:01.668598385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:44130"
ts=2026-04-23T12:33:04.753921725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40092"
ts=2026-04-23T12:33:12.207345484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:33:30.563727902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:33:32.880842052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:55678"
ts=2026-04-23T12:33:42.210321395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3d8887a3 10.130.0.62:7946"
ts=2026-04-23T12:34:12.212356941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:34:22.274247678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:34:22.276353572Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:34:29.924311517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:51918"
ts=2026-04-23T12:34:42.214089563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:34:59.926871487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:39614"
ts=2026-04-23T12:35:04.764463918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52500"
level=debug ts=2026-04-23T12:35:07.536463295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:35:12.216419941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:35:34.766976457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48984"
ts=2026-04-23T12:35:42.218589586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:35:45.705566265Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3d8887a3 (timeout reached)"
ts=2026-04-23T12:35:48.705822373Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3d8887a3 has failed, no acks received"
ts=2026-04-23T12:35:53.706415691Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-3d8887a3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:36:02.897275135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:58398"
ts=2026-04-23T12:36:04.769590443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40424"
level=debug ts=2026-04-23T12:36:09.604125203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:36:12.221294197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:36:32.898652264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:47472"
ts=2026-04-23T12:36:42.223902209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:36:50.631519297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35558"
ts=2026-04-23T12:36:50.643169865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35570"
ts=2026-04-23T12:37:04.774240456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48458"
level=debug ts=2026-04-23T12:37:10.190542338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:37:12.226505306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:37:34.776029541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54158"
ts=2026-04-23T12:37:42.228859169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:38:02.906338971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:37258"
ts=2026-04-23T12:38:12.230076403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:38:12.271040007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:38:23.439636505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:38588"
level=error ts=2026-04-23T12:38:34.978733235Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:38:34.980576591Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:38:34.982390036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:38:34.982417846Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:38:42.231494068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:38:50.615429119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:38:50.617168686Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:39:10.705299323Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-598355d3 (timeout reached)"
ts=2026-04-23T12:39:12.234292294Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-598355d3 failed: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-04-23T12:39:13.706140019Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-598355d3 has failed, no acks received"
ts=2026-04-23T12:39:15.706927857Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-598355d3 (timeout reached)"
ts=2026-04-23T12:39:18.707164008Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-598355d3 has failed, no acks received"
ts=2026-04-23T12:39:32.913025775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:51930"
ts=2026-04-23T12:39:33.706715856Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-598355d3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:39:34.788327644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43162"
ts=2026-04-23T12:39:35.706541931Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-598355d3 (timeout reached)"
ts=2026-04-23T12:39:38.7062401Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-598355d3 has failed, no acks received"
level=debug ts=2026-04-23T12:39:40.412614001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:39:42.235433561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:40:04.790969616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51102"
ts=2026-04-23T12:40:12.237565844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:40:32.916015269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39114"
level=debug ts=2026-04-23T12:40:33.365070847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:40:42.239481749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:41:12.241222085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:41:16.394992448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:41:42.242404022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:41:58.620210165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:48422"
ts=2026-04-23T12:41:58.631495575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:48434"
level=debug ts=2026-04-23T12:41:58.886923749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:42:12.244689387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ca34acd0 10.130.0.62:7946"
ts=2026-04-23T12:42:32.924226245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:51304"
level=debug ts=2026-04-23T12:42:34.697252673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:42:34.698856529Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:42:34.802486478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46984"
ts=2026-04-23T12:42:42.246293025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:43:04.80418998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50942"
ts=2026-04-23T12:43:12.247474476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:43:28.690381634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:46230"
level=debug ts=2026-04-23T12:43:33.99582887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:43:34.80620136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44474"
ts=2026-04-23T12:43:40.705604471Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ca34acd0 (timeout reached)"
ts=2026-04-23T12:43:42.249167887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:43:43.705829374Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ca34acd0 has failed, no acks received"
ts=2026-04-23T12:43:55.706119932Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ca34acd0 (timeout reached)"
ts=2026-04-23T12:43:57.009289888Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ca34acd0 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:43:58.706715192Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ca34acd0 has failed, no acks received"
ts=2026-04-23T12:44:04.808465112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40540"
ts=2026-04-23T12:44:12.25060505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:44:19.57534776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:44:32.93173957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39342"
ts=2026-04-23T12:44:42.251973533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:45:02.9332885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:37222"
ts=2026-04-23T12:45:04.813781775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52160"
ts=2026-04-23T12:45:12.253673716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:45:13.519816331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:45:32.934581378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:48588"
ts=2026-04-23T12:45:34.816192753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58156"
ts=2026-04-23T12:45:42.255279102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:46:04.818345507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57704"
ts=2026-04-23T12:46:12.256255564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:46:14.37820034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:46:32.938452661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:48070"
ts=2026-04-23T12:46:42.25841138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:46:57.677150441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:46:57.679241437Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:47:02.939690741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:49136"
ts=2026-04-23T12:47:12.261025049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:47:32.940725971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:54234"
level=debug ts=2026-04-23T12:47:37.958184552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:47:42.263557518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:48:02.941999176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:46148"
ts=2026-04-23T12:48:12.265365387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:48:20.102053899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:48:32.94755087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:44794"
ts=2026-04-23T12:48:33.626351913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:58556"
ts=2026-04-23T12:48:33.637838331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:58558"
ts=2026-04-23T12:48:34.828549441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36398"
level=error ts=2026-04-23T12:48:34.978298166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:48:34.980351057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:48:34.982240861Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:48:34.982269128Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:48:42.267291615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:49:04.830796138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43154"
ts=2026-04-23T12:49:09.135630586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:34328"
level=debug ts=2026-04-23T12:49:12.265036855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:49:12.269201702Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-562a5aa3 failed: dial tcp 10.130.0.62:7946: connect: connection refused"
ts=2026-04-23T12:49:20.705502669Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-562a5aa3 (timeout reached)"
ts=2026-04-23T12:49:23.705132229Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-562a5aa3 has failed, no acks received"
ts=2026-04-23T12:49:32.951901657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:55262"
ts=2026-04-23T12:49:34.832462076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43248"
ts=2026-04-23T12:49:40.705290848Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-562a5aa3 (timeout reached)"
ts=2026-04-23T12:49:42.099218409Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-562a5aa3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:49:42.270813839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:49:43.705405548Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-562a5aa3 has failed, no acks received"
level=debug ts=2026-04-23T12:49:49.532941911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:50:04.835208454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35714"
ts=2026-04-23T12:50:12.2723099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:50:31.125056241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:50:31.126805492Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:50:34.837371075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44868"
ts=2026-04-23T12:50:42.274258046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:51:04.839481267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57098"
ts=2026-04-23T12:51:12.275885072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:51:22.661644008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:51:34.841932888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40060"
ts=2026-04-23T12:51:42.277461832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:51:59.820949705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:52:12.27929797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:52:42.281466512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:52:46.230418615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:53:02.967457943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:34302"
ts=2026-04-23T12:53:12.28346846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:53:32.969323253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:57734"
level=debug ts=2026-04-23T12:53:33.093943199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:53:42.285074519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:54:02.970929392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:47420"
ts=2026-04-23T12:54:04.852353025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44656"
ts=2026-04-23T12:54:12.286293907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:54:15.618988822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47046"
ts=2026-04-23T12:54:15.6305771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47062"
level=debug ts=2026-04-23T12:54:25.937179726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:54:25.938812803Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:54:42.288509007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-26abdda4 10.130.0.62:7946"
ts=2026-04-23T12:54:57.135955162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47224"
ts=2026-04-23T12:55:10.705455686Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-26abdda4 (timeout reached)"
ts=2026-04-23T12:55:12.290694354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:55:13.705234242Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-26abdda4 has failed, no acks received"
level=debug ts=2026-04-23T12:55:25.947824982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:55:28.700596868Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-26abdda4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T12:55:32.978057796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:52794"
ts=2026-04-23T12:55:42.292416069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:56:02.979753504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:36876"
level=debug ts=2026-04-23T12:56:11.647929685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:56:12.293685111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:56:32.980998802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:35810"
ts=2026-04-23T12:56:34.864864236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50726"
ts=2026-04-23T12:56:42.295308804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:56:46.125591057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:57:12.296908415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T12:57:34.868700104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43818"
level=debug ts=2026-04-23T12:57:36.658455932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:57:42.298480741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:58:12.300036553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T12:58:21.872347029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T12:58:21.873961026Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:58:34.873619229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57992"
level=error ts=2026-04-23T12:58:34.977961203Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:58:34.979969312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:58:34.98162354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T12:58:34.981648405Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T12:58:42.302115465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:58:57.161141746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T12:59:02.989618405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:49430"
ts=2026-04-23T12:59:04.875856725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41042"
ts=2026-04-23T12:59:12.302989978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T12:59:32.991069491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:47064"
ts=2026-04-23T12:59:34.878617521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58652"
ts=2026-04-23T12:59:42.303865064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T12:59:46.296569215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:00:02.9919831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:52110"
ts=2026-04-23T13:00:04.881180415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39084"
ts=2026-04-23T13:00:06.639554545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43026"
ts=2026-04-23T13:00:06.651489894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43034"
ts=2026-04-23T13:00:12.306284144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8edb5818 10.130.0.62:7946"
level=debug ts=2026-04-23T13:00:31.738772162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:00:34.883584542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60288"
ts=2026-04-23T13:00:35.705463569Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8edb5818 (timeout reached)"
ts=2026-04-23T13:00:38.705166371Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8edb5818 has failed, no acks received"
ts=2026-04-23T13:00:42.308649746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:00:55.705134518Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8edb5818 (timeout reached)"
ts=2026-04-23T13:00:58.705589041Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8edb5818 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T13:00:58.705628302Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8edb5818 has failed, no acks received"
ts=2026-04-23T13:01:12.309795717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T13:01:20.492203315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:01:32.998437803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:44466"
ts=2026-04-23T13:01:34.887732039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54264"
ts=2026-04-23T13:01:42.311238261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:02:03.000284918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:39282"
ts=2026-04-23T13:02:04.889843525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43014"
ts=2026-04-23T13:02:12.312841188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:02:18.514192315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:02:18.51673243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:02:42.314378365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:03:03.004349169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:57172"
level=debug ts=2026-04-23T13:03:08.139730835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:03:12.315956276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:03:34.897459619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59354"
ts=2026-04-23T13:03:42.316792792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T13:04:04.963722285Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:04:12.318438528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:04:33.009868774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:52298"
ts=2026-04-23T13:04:42.320356596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:04:50.680977023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:05:12.322606645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:05:28.619290958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53288"
ts=2026-04-23T13:05:28.630506395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53292"
ts=2026-04-23T13:05:33.013947429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:40136"
ts=2026-04-23T13:05:35.705245352Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0b8be9d8 (timeout reached)"
level=debug ts=2026-04-23T13:05:37.351208764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:05:38.705154616Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0b8be9d8 has failed, no acks received"
ts=2026-04-23T13:05:42.323786371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:05:58.70536567Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-0b8be9d8 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-23T13:06:00.705898788Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0b8be9d8 (timeout reached)"
ts=2026-04-23T13:06:03.01556047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:34650"
ts=2026-04-23T13:06:03.705392681Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0b8be9d8 has failed, no acks received"
ts=2026-04-23T13:06:12.326098551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:06:27.831211448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:06:27.833079954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:06:42.327559209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:07:03.019720677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:38092"
ts=2026-04-23T13:07:04.914213595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42946"
ts=2026-04-23T13:07:12.328771173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T13:07:19.051482108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:07:42.330770846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:08:12.33267493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:08:13.68103594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:08:34.921528278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39146"
level=error ts=2026-04-23T13:08:34.981036632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:08:34.983140803Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:08:34.984845973Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-23T13:08:34.98487054Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:08:42.33399963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:09:04.923421455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59614"
level=debug ts=2026-04-23T13:09:09.33439405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:09:12.335007632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:09:33.028908131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:55062"
ts=2026-04-23T13:09:34.925253675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49526"
ts=2026-04-23T13:09:42.336061548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T13:09:58.843333453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:10:03.030374574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:46192"
ts=2026-04-23T13:10:04.926950179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36046"
ts=2026-04-23T13:10:12.337934791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:10:33.031467805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:41492"
ts=2026-04-23T13:10:35.621161342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:58586"
ts=2026-04-23T13:10:35.633406549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:58592"
ts=2026-04-23T13:10:42.341068809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-788a7ae3 10.130.0.62:7946"
level=debug ts=2026-04-23T13:10:58.140995303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:10:58.142534521Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:11:04.931912775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58320"
ts=2026-04-23T13:11:05.705869843Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-788a7ae3 (timeout reached)"
ts=2026-04-23T13:11:08.705779017Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-788a7ae3 has failed, no acks received"
ts=2026-04-23T13:11:12.344342182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:11:18.699876698Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-788a7ae3 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-23T13:11:39.50820226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:11:42.346620837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:12:03.037648509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:34322"
ts=2026-04-23T13:12:12.348853629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:12:20.542638821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:12:33.039455236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:37668"
ts=2026-04-23T13:12:42.351134951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:13:03.040685957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:38704"
ts=2026-04-23T13:13:04.940984503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55788"
ts=2026-04-23T13:13:12.353010551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:13:13.456716853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:13:33.042593936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:43880"
ts=2026-04-23T13:13:42.355637102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:13:49.304245262Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:14:03.043515836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:36688"
ts=2026-04-23T13:14:12.357788512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
level=debug ts=2026-04-23T13:14:42.263092937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-23T13:14:42.2648006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-23T13:14:42.360061646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:15:04.949989883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53932"
ts=2026-04-23T13:15:12.362502887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:15:31.855609678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:15:34.951892016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52766"
ts=2026-04-23T13:15:42.364258526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
ts=2026-04-23T13:16:04.628494343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37414"
ts=2026-04-23T13:16:04.640221968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37430"
ts=2026-04-23T13:16:12.366440837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3406647d 10.130.0.62:7946"
level=debug ts=2026-04-23T13:16:29.136845814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:16:30.70653288Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3406647d (timeout reached)"
ts=2026-04-23T13:16:33.052658226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:36924"
ts=2026-04-23T13:16:33.706142463Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3406647d has failed, no acks received"
ts=2026-04-23T13:16:34.956029246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51192"
ts=2026-04-23T13:16:35.706873889Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3406647d (timeout reached)"
ts=2026-04-23T13:16:38.706941835Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3406647d has failed, no acks received"
ts=2026-04-23T13:16:42.368690506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"
ts=2026-04-23T13:17:03.053973135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:57240"
ts=2026-04-23T13:17:12.370802486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zg6s2-be750081 10.129.0.70:7946"
level=debug ts=2026-04-23T13:17:31.257683267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-23T13:17:33.055177829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.94:41836"
ts=2026-04-23T13:17:34.960087866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38392"
ts=2026-04-23T13:17:42.372672577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nzf24-6b05b3d3 10.128.0.94:7946"