level=info ts=2026-04-19T13:28:35.531172336Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T13:28:35.531192498Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T13:28:35.532390909Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T13:28:35.532657836Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T13:28:35.53271836Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=debug ts=2026-04-19T13:28:35.533097662Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T13:28:35.53315384Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=info ts=2026-04-19T13:28:35.53322923Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-5b6d7a20
ts=2026-04-19T13:28:35.533313812Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T13:28:35.533327336Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=debug ts=2026-04-19T13:28:35.533348323Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=info ts=2026-04-19T13:28:35.533504963Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T13:28:35.541001438Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T13:28:35.541021689Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T13:28:35.54105048Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.547907ms
level=error ts=2026-04-19T13:28:35.541061345Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T13:28:35.54107716Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T13:28:35.544470561Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T13:28:35.544486879Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T13:28:35.548855264Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-19T13:28:35.548878682Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-19T13:28:35.548873375Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-19T13:28:35.548898762Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T13:28:35.548966427Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-19T13:28:35.548975593Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-19T13:28:35.549061786Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-19T13:28:35.549083351Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T13:28:35.549187461Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-19T13:28:35.549198706Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T13:28:35.549239675Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T13:28:35.549235562Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T13:28:35.549328397Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T13:28:35.549499487Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T13:28:35.728954848Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T13:28:35.729069941Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T13:28:36.729762965Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T13:28:36.840213772Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T13:28:36.840289977Z caller=loki.go:599 msg="Loki started" startup_time=1.374476219s
ts=2026-04-19T13:28:37.456640494Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T13:28:37.456666892Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:28:41.404182101Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T13:28:41.404212117Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T13:28:41.840441742Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T13:28:41.840519952Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T13:28:47.885786534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
ts=2026-04-19T13:28:47.888477683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
level=info ts=2026-04-19T13:28:47.889581932Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=12.34849809s
level=debug ts=2026-04-19T13:28:56.335224181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:29:02.212734038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41168"
level=debug ts=2026-04-19T13:29:13.748685788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:29:16.700694219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:29:20.4095623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58546"
ts=2026-04-19T13:29:20.692252962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41812"
level=debug ts=2026-04-19T13:29:31.019769633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:29:46.702434681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:29:54.761710405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:30:16.703386716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:30:20.4146187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35386"
level=debug ts=2026-04-19T13:30:30.53285312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T13:30:30.534787427Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:30:46.705254185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:30:53.558370367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48262"
level=debug ts=2026-04-19T13:31:11.658593732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:31:16.708275299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:31:20.418867028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46752"
ts=2026-04-19T13:31:46.710642888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:32:16.712303671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:32:20.422929819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59394"
ts=2026-04-19T13:32:23.564446047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54546"
level=debug ts=2026-04-19T13:32:28.480934859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:32:46.715546581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:32:50.706260609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51998"
ts=2026-04-19T13:33:16.718422969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:33:20.426556307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55286"
ts=2026-04-19T13:33:23.568764066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52322"
level=debug ts=2026-04-19T13:33:29.857260517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:33:46.720799856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:33:53.570637298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50436"
ts=2026-04-19T13:34:16.723404482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:34:20.712883853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50222"
level=debug ts=2026-04-19T13:34:44.079532838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T13:34:46.72539872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:34:50.43325911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53298"
ts=2026-04-19T13:35:16.727742886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:35:20.435926807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52428"
ts=2026-04-19T13:35:46.730617307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:35:50.720101856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54854"
level=debug ts=2026-04-19T13:36:01.38171028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T13:36:01.383450518Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:36:16.732077813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:36:23.582836587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41924"
ts=2026-04-19T13:36:46.733126272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:36:48.09716315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:36:50.725509679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47252"
ts=2026-04-19T13:37:16.734646995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:37:40.492084589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:37:46.736403845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:38:16.738803799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:38:20.730875872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56282"
ts=2026-04-19T13:38:23.59324076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58352"
level=debug ts=2026-04-19T13:38:39.503386701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-19T13:38:41.840625507Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T13:38:41.842700736Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:38:41.844949361Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:38:41.846511864Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:38:41.846538325Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T13:38:41.846551502Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T13:38:46.741748644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:38:50.73378084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54686"
ts=2026-04-19T13:39:16.744177757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:39:23.597055054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42976"
level=debug ts=2026-04-19T13:39:33.04214987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:39:46.746351845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T13:40:13.778754096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:40:13.780497484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:40:16.749462703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:40:46.752217684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:40:56.393828603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:41:16.753644452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T13:41:44.369966142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:41:46.755315928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:41:50.459140308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60522"
ts=2026-04-19T13:42:16.756484079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:42:20.462159737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56208"
ts=2026-04-19T13:42:23.611588711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47272"
level=debug ts=2026-04-19T13:42:33.105833019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:42:46.75763381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:42:50.750017041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59684"
ts=2026-04-19T13:43:16.75964273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:43:20.752221417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42756"
level=debug ts=2026-04-19T13:43:30.3190868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:43:46.762000306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:43:50.469383325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58444"
ts=2026-04-19T13:43:53.618063711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34116"
level=debug ts=2026-04-19T13:44:13.844406185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:44:13.846202595Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:44:16.764220687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:44:20.472237341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41896"
ts=2026-04-19T13:44:23.619411619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50870"
ts=2026-04-19T13:44:46.766596108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T13:45:01.472483555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:45:16.769074333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:45:20.477870135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40318"
level=debug ts=2026-04-19T13:45:36.527224187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:45:46.77140662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:45:50.481454421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47200"
ts=2026-04-19T13:45:50.762659872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35430"
ts=2026-04-19T13:45:53.626356552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57752"
ts=2026-04-19T13:46:16.772441549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:46:28.718166186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:46:46.773715981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:46:50.765508219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54616"
level=debug ts=2026-04-19T13:47:15.330896748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:47:16.775820578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:47:20.768161434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45092"
ts=2026-04-19T13:47:46.779007552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:47:50.77040968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47148"
level=debug ts=2026-04-19T13:47:54.509968182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:47:54.511556964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:48:16.780616348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:48:20.493198925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59830"
level=debug ts=2026-04-19T13:48:33.030382796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:48:41.849903569Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:48:41.851570902Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:48:41.853549922Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:48:41.853574547Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:48:46.782862067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:49:16.785951085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:49:23.643742692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41928"
level=debug ts=2026-04-19T13:49:34.904097298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:49:46.788708681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:49:50.499750782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43404"
ts=2026-04-19T13:49:50.779478787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44390"
ts=2026-04-19T13:49:53.645369696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58270"
ts=2026-04-19T13:50:16.79088405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:50:31.842596292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:50:46.793065648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:50:50.504430273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50424"
ts=2026-04-19T13:51:16.79584019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:51:20.507368644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42816"
level=debug ts=2026-04-19T13:51:22.035240843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:51:46.797991575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:51:50.510234285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49720"
level=debug ts=2026-04-19T13:52:04.875857266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:52:04.877536454Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:52:16.799422889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:52:23.657517218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50782"
ts=2026-04-19T13:52:46.802054478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:52:50.791030484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43420"
level=debug ts=2026-04-19T13:53:05.090216796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:53:16.804568153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:53:43.904853046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:53:46.806361877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:54:16.809582911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T13:54:20.94758533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:54:46.811678774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:54:50.800189121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56300"
ts=2026-04-19T13:55:16.813741163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T13:55:17.553433453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:55:20.80239142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44804"
ts=2026-04-19T13:55:23.671955759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49722"
ts=2026-04-19T13:55:46.8168329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:55:50.527809739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41938"
level=debug ts=2026-04-19T13:55:54.984589074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:55:54.986942676Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:56:16.819253313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T13:56:23.676582881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45540"
level=debug ts=2026-04-19T13:56:36.516478189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:56:46.820820706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:57:16.822772235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
level=debug ts=2026-04-19T13:57:18.182440606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:57:20.536544517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52262"
ts=2026-04-19T13:57:46.825011953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T13:58:09.535801319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:58:16.828297641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=error ts=2026-04-19T13:58:41.852791856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:58:41.854737628Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:58:41.856764714Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T13:58:41.856798469Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:58:46.831397566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T13:58:50.545293121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35464"
level=debug ts=2026-04-19T13:58:54.30682731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T13:59:16.834102666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T13:59:20.54824977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53308"
ts=2026-04-19T13:59:23.691566155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53176"
level=debug ts=2026-04-19T13:59:42.998557265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T13:59:43.000217746Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T13:59:46.836872268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T14:00:16.839181343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:00:20.556076785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51986"
ts=2026-04-19T14:00:20.822941758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56426"
ts=2026-04-19T14:00:23.695788621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43188"
level=debug ts=2026-04-19T14:00:28.266921397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:00:46.843915682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:00:50.826172462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48064"
level=debug ts=2026-04-19T14:01:11.000129152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:01:16.845814261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T14:01:46.848023805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T14:02:12.77451934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:02:16.850457837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:02:46.853029734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:02:50.833885878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37940"
level=debug ts=2026-04-19T14:03:11.244990973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:03:16.855513167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8f1e5742 10.129.0.81:7946"
ts=2026-04-19T14:03:23.71227103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56864"
ts=2026-04-19T14:03:46.858069664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:03:50.572714263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34256"
ts=2026-04-19T14:03:50.837830267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39196"
ts=2026-04-19T14:03:53.714343127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57146"
level=debug ts=2026-04-19T14:03:59.265422924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:03:59.267313068Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:04:05.828468097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48412"
ts=2026-04-19T14:04:05.840593128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48426"
ts=2026-04-19T14:04:07.534660414Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8f1e5742 (timeout reached)"
ts=2026-04-19T14:04:10.534378816Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8f1e5742 has failed, no acks received"
ts=2026-04-19T14:04:16.861367253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c22883b 10.129.0.81:7946"
ts=2026-04-19T14:04:30.535969664Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8f1e5742 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T14:04:32.533864253Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8f1e5742 (timeout reached)"
ts=2026-04-19T14:04:35.534327491Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8f1e5742 has failed, no acks received"
ts=2026-04-19T14:04:46.866728696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:04:50.843425493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52832"
ts=2026-04-19T14:04:56.528366616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58508"
level=debug ts=2026-04-19T14:04:59.27101255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:05:16.869446985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:05:20.579893522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52560"
ts=2026-04-19T14:05:46.871488192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c22883b 10.129.0.81:7946"
level=debug ts=2026-04-19T14:05:50.769417477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:05:56.531990192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41648"
ts=2026-04-19T14:06:16.873791541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:06:20.854179994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43742"
level=debug ts=2026-04-19T14:06:40.7652186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:06:46.87675802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:06:50.586657052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39224"
level=debug ts=2026-04-19T14:07:16.40594392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:07:16.879447744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:07:46.882772118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:07:51.566293686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:07:51.568397138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:07:56.541555211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48374"
ts=2026-04-19T14:08:16.88541081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:08:20.862099963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53870"
level=debug ts=2026-04-19T14:08:27.025291762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:08:41.851073316Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:08:41.852897037Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:08:41.854761648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:08:41.85478838Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:08:46.888453266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:08:50.594754473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59578"
level=debug ts=2026-04-19T14:09:09.524051973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:09:16.891475964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:09:46.894598298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T14:09:47.20205213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:09:50.600163363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55428"
ts=2026-04-19T14:10:16.896100049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6c22883b 10.129.0.81:7946"
level=debug ts=2026-04-19T14:10:24.054872594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:10:27.53407737Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6c22883b (timeout reached)"
ts=2026-04-19T14:10:30.534178783Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6c22883b has failed, no acks received"
ts=2026-04-19T14:10:42.544452621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34122"
ts=2026-04-19T14:10:42.551655669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34130"
ts=2026-04-19T14:10:46.898366803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:10:50.534370272Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-6c22883b as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T14:10:52.534339106Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6c22883b (timeout reached)"
ts=2026-04-19T14:10:55.534581026Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6c22883b has failed, no acks received"
level=debug ts=2026-04-19T14:11:07.202768408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:11:07.204438852Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:11:16.901297845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T14:11:42.562612842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:11:46.903543371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:12:03.272905539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47884"
ts=2026-04-19T14:12:16.906604722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:12:42.498513986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:12:46.909121353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:12:50.614849248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33148"
ts=2026-04-19T14:13:16.911502545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:13:20.617985544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45006"
ts=2026-04-19T14:13:20.88014955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33482"
level=debug ts=2026-04-19T14:13:32.419472579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:13:46.91416179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:13:50.620624003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51328"
ts=2026-04-19T14:14:16.916526146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297a71c4 10.129.0.81:7946"
ts=2026-04-19T14:14:33.285541087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45810"
level=debug ts=2026-04-19T14:14:33.57793385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:14:46.918798607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:15:03.286918127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56950"
ts=2026-04-19T14:15:16.921771537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:15:24.191323236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:15:24.193274924Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:15:46.924106132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:15:50.895062546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43644"
level=debug ts=2026-04-19T14:16:05.350701383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:16:16.926852716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:16:44.474572243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:16:46.929290407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:17:16.931716758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:17:20.639468193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50264"
ts=2026-04-19T14:17:20.914819736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52870"
level=debug ts=2026-04-19T14:17:23.426404172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:17:46.934723298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:17:50.641880172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36520"
ts=2026-04-19T14:17:50.918315891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57690"
ts=2026-04-19T14:18:03.30765916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51134"
level=debug ts=2026-04-19T14:18:12.473575237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:18:16.937018208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:18:20.644185775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46542"
ts=2026-04-19T14:18:20.920913463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34588"
level=error ts=2026-04-19T14:18:41.850699549Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:18:41.852669908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:18:41.855270307Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:18:41.855293978Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:18:46.940006711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:18:50.923279113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34924"
ts=2026-04-19T14:19:03.31115118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42764"
level=debug ts=2026-04-19T14:19:14.021644811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:19:14.023462434Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:19:16.941627537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-297a71c4 10.129.0.81:7946"
ts=2026-04-19T14:19:46.944170353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:19:55.145605587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:20:07.534823674Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-297a71c4 (timeout reached)"
ts=2026-04-19T14:20:10.534614216Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-297a71c4 has failed, no acks received"
ts=2026-04-19T14:20:16.947132371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:20:18.55123263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44018"
ts=2026-04-19T14:20:18.558473021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44030"
ts=2026-04-19T14:20:20.928505671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34842"
ts=2026-04-19T14:20:30.53499535Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-297a71c4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T14:20:32.534074056Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-297a71c4 (timeout reached)"
ts=2026-04-19T14:20:35.534450641Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-297a71c4 has failed, no acks received"
level=debug ts=2026-04-19T14:20:36.195970633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:20:46.950113159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:20:50.655649425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47792"
ts=2026-04-19T14:21:16.953601687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:21:20.659183867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55568"
level=debug ts=2026-04-19T14:21:34.371011783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:21:37.411504769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58568"
ts=2026-04-19T14:21:46.961184009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:22:07.412786969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48694"
ts=2026-04-19T14:22:16.966582667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
level=debug ts=2026-04-19T14:22:25.269746451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:22:37.414519624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60642"
ts=2026-04-19T14:22:46.96951295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:22:50.667432644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43066"
ts=2026-04-19T14:22:50.938258846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59592"
ts=2026-04-19T14:23:07.415281433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38156"
ts=2026-04-19T14:23:16.972467764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:23:26.879871788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:23:26.881412543Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:23:37.416905167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50998"
ts=2026-04-19T14:23:46.974829414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef2a1d4 10.129.0.81:7946"
ts=2026-04-19T14:23:50.94463771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36780"
ts=2026-04-19T14:24:16.976545507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:24:20.674364979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53646"
ts=2026-04-19T14:24:20.946603058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39616"
level=debug ts=2026-04-19T14:24:23.526023196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:24:46.978775926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:25:16.981160558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef2a1d4 10.129.0.81:7946"
level=debug ts=2026-04-19T14:25:22.336633699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:25:37.426167946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59434"
ts=2026-04-19T14:25:46.983446921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:25:50.952698471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56158"
ts=2026-04-19T14:26:16.986528102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wnp9c-43166b0f 10.130.0.69:7946"
ts=2026-04-19T14:26:20.954816517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52024"
level=debug ts=2026-04-19T14:26:21.305351023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:26:37.430393321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38796"
ts=2026-04-19T14:26:46.988099045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef2a1d4 10.129.0.81:7946"
ts=2026-04-19T14:26:50.957893407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33330"
ts=2026-04-19T14:27:07.431747532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58752"
level=debug ts=2026-04-19T14:27:12.665626001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:27:16.990524982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
ts=2026-04-19T14:27:20.687145073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59082"
ts=2026-04-19T14:27:46.992760893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-kwcjj-47f7217e 10.130.0.68:7946"
level=debug ts=2026-04-19T14:27:48.46082654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:27:48.462394001Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:28:16.994850067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef2a1d4 10.129.0.81:7946"
ts=2026-04-19T14:28:20.690677569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49120"
level=debug ts=2026-04-19T14:28:32.645632506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:28:41.849374844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:28:41.851266955Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:28:41.853970715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:28:41.853996079Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:28:46.996247475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4ef2a1d4 10.129.0.81:7946"
ts=2026-04-19T14:28:50.693074345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:32836"