level=info ts=2026-04-16T18:45:08.73863373Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T18:45:08.738657262Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-16T18:45:08.739455559Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T18:45:08.739890802Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T18:45:08.740002417Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.73 inf=eth0
level=debug ts=2026-04-16T18:45:08.74127944Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T18:45:08.7413708Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.73 inf=eth0
level=info ts=2026-04-16T18:45:08.74147525Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-c8aff13b
ts=2026-04-16T18:45:08.741562316Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T18:45:08.741580341Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=debug ts=2026-04-16T18:45:08.741603237Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.73 advertisePort=7946
level=info ts=2026-04-16T18:45:08.741775112Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-16T18:45:08.747633749Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T18:45:08.747650203Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T18:45:08.74766096Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.888555ms
level=error ts=2026-04-16T18:45:08.747684019Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-16T18:45:08.747697951Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-16T18:45:08.750133916Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T18:45:08.750146217Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T18:45:08.754827493Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-16T18:45:08.754845476Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-16T18:45:08.754856654Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-16T18:45:08.754881653Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-16T18:45:08.754895604Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T18:45:08.754917498Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-16T18:45:08.75493952Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-16T18:45:08.755102653Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-16T18:45:08.75511179Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-16T18:45:08.755115509Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-16T18:45:08.755150461Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-16T18:45:08.755180748Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-16T18:45:08.755197407Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T18:45:08.755334811Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-16T18:45:08.755342227Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-16T18:45:08.755352588Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-16T18:45:09.756162341Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-04-16T18:45:09.867043731Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T18:45:09.867076506Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T18:45:09.897165401Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-16T18:45:09.897218317Z caller=loki.go:599 msg="Loki started" startup_time=1.224569169s
ts=2026-04-16T18:45:12.055894429Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T18:45:12.0559221Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T18:45:14.89726054Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-16T18:45:14.897330044Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-16T18:45:16.423934547Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T18:45:16.423962847Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T18:45:23.506275953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:45:26.415728728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.53:7946"
ts=2026-04-16T18:45:26.417245894Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.55:7946: dial tcp 10.128.0.55:7946: connect: connection refused"
ts=2026-04-16T18:45:26.418080047Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.72:7946: dial tcp 10.130.0.72:7946: connect: connection refused"
ts=2026-04-16T18:45:26.418185224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-04-16T18:45:26.418208585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42056"
level=info ts=2026-04-16T18:45:26.418549746Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=17.670846152s
ts=2026-04-16T18:45:27.792352478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:54438"
ts=2026-04-16T18:45:30.777588149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36084"
ts=2026-04-16T18:45:37.78905314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42456"
level=debug ts=2026-04-16T18:45:42.507358207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:46:01.814213247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T18:46:08.079614108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T18:46:27.649723145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:46:31.81643815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:46:50.230988534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53762"
level=debug ts=2026-04-16T18:46:59.625471288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T18:46:59.627505323Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T18:47:01.820034761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:47:30.10520126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:34184"
ts=2026-04-16T18:47:31.823059112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T18:47:35.421057186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:48:01.825761814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T18:48:20.238482751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55030"
level=debug ts=2026-04-16T18:48:28.761959294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:48:31.828506986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T18:48:49.411400975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40056"
ts=2026-04-16T18:48:50.241491249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59812"
ts=2026-04-16T18:49:01.831704594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:49:30.116318382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:35580"
ts=2026-04-16T18:49:31.835992216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T18:49:43.938821807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:50:01.839341843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:50:19.421227496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45136"
ts=2026-04-16T18:50:31.841853559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T18:50:48.332268821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T18:50:49.423908031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38304"
ts=2026-04-16T18:51:00.125327506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:60604"
ts=2026-04-16T18:51:01.844339735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T18:51:30.127899721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:44174"
ts=2026-04-16T18:51:31.847204524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T18:51:46.655599702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T18:51:46.657340698Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T18:52:01.850768273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:52:19.429345729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48770"
level=debug ts=2026-04-16T18:52:25.157964164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:52:31.855372192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T18:53:01.858531669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T18:53:26.627217051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:53:30.139539888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:36440"
ts=2026-04-16T18:53:31.861912266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T18:54:00.14123841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:52322"
ts=2026-04-16T18:54:01.865481786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T18:54:14.970943927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:54:19.439164297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:48564"
ts=2026-04-16T18:54:20.26629688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37650"
ts=2026-04-16T18:54:31.868056465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T18:54:56.502626258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:55:01.871411925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=info ts=2026-04-16T18:55:14.897823486Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-16T18:55:14.901815465Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T18:55:14.903937968Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T18:55:14.905868528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T18:55:14.905906301Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T18:55:14.905921116Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-16T18:55:31.873863549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T18:55:40.28870187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T18:55:40.290561858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T18:55:49.44880337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:53302"
ts=2026-04-16T18:55:50.273215612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56846"
ts=2026-04-16T18:56:00.153495012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:43748"
ts=2026-04-16T18:56:01.876521698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T18:56:17.939614818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:56:20.275769508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42380"
ts=2026-04-16T18:56:31.88019065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T18:56:49.454365195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44874"
ts=2026-04-16T18:57:00.158953227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:49090"
ts=2026-04-16T18:57:01.883470211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T18:57:14.808179195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:57:30.161891183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:37724"
ts=2026-04-16T18:57:31.88635533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T18:57:49.460149167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:37256"
level=debug ts=2026-04-16T18:57:52.434685554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:58:01.888708898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T18:58:30.166480034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:36506"
ts=2026-04-16T18:58:31.891740546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T18:58:43.843694667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T18:58:49.46692745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35892"
ts=2026-04-16T18:58:50.289819121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39162"
ts=2026-04-16T18:59:01.894778042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T18:59:20.292827436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58560"
ts=2026-04-16T18:59:31.897871483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T18:59:34.389132542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T18:59:34.391095004Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T18:59:49.471178485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33326"
ts=2026-04-16T19:00:01.900450333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:00:18.725506949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:00:30.177326266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:41922"
ts=2026-04-16T19:00:31.903919296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:00:49.477658387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38950"
level=debug ts=2026-04-16T19:00:56.138559651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:01:01.907052995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:01:19.481175265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56952"
ts=2026-04-16T19:01:30.183280338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:52378"
ts=2026-04-16T19:01:31.908933048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:01:46.569487522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:02:01.911296045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:02:20.302190988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47358"
level=debug ts=2026-04-16T19:02:23.507576769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:02:30.189121722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:40616"
ts=2026-04-16T19:02:31.914284046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T19:03:01.91691746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:03:19.490008715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33534"
level=debug ts=2026-04-16T19:03:20.282483443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:03:20.284303503Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:03:31.919324508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:04:01.922005366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:04:11.565446865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:04:31.925574177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:05:01.928126622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:05:11.701454155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:05:14.90922525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:05:14.911053953Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:05:14.912840666Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:05:14.912866632Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:05:19.499856349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49526"
ts=2026-04-16T19:05:30.206692606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:44398"
ts=2026-04-16T19:05:31.930783472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:05:49.503312001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:51422"
ts=2026-04-16T19:06:01.934107314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:06:06.226076016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:06:30.211823084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:60462"
ts=2026-04-16T19:06:31.93720101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:06:56.067207619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:07:00.213587337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:34120"
ts=2026-04-16T19:07:01.939769138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:07:31.942291919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:07:40.842918214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:07:40.844605497Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:08:00.218567272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:55210"
ts=2026-04-16T19:08:01.945169099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:08:22.168621573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:08:31.948351818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T19:09:01.951550372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:09:06.587616897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:09:19.517828838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60614"
ts=2026-04-16T19:09:30.227688205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:59788"
ts=2026-04-16T19:09:31.955196628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:09:44.74088877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:09:50.335660606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45210"
ts=2026-04-16T19:10:01.957661419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:10:31.960413095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:10:42.5506104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:11:01.963583195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:11:18.049713383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:11:18.051546821Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:11:31.966874114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:11:50.345784106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58350"
ts=2026-04-16T19:12:01.970369417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:12:03.299839433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:12:31.973238246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:12:39.20084969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:12:50.351580714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52816"
ts=2026-04-16T19:13:01.975750901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:13:24.001752934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:13:31.978711451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:13:50.35856062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37116"
ts=2026-04-16T19:14:01.981577818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T19:14:20.361598994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44288"
level=debug ts=2026-04-16T19:14:21.271269779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:14:30.261101721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:44678"
ts=2026-04-16T19:14:31.985238764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:14:50.364826295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50056"
ts=2026-04-16T19:15:00.263798396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:36626"
ts=2026-04-16T19:15:01.992841367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:15:04.840987086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:15:04.842812527Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:15:14.911004601Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:15:14.913207627Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:15:14.915201721Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:15:14.915247324Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:15:30.2659619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:34790"
ts=2026-04-16T19:15:31.998295316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:15:50.370716522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42596"
level=debug ts=2026-04-16T19:15:52.996216557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:16:02.001015529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
ts=2026-04-16T19:16:20.373912051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38618"
ts=2026-04-16T19:16:32.003614475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:16:53.386514865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:17:02.00648402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:17:19.561018018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43298"
ts=2026-04-16T19:17:30.278227041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:46258"
ts=2026-04-16T19:17:32.00963507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:17:50.382022142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44886"
level=debug ts=2026-04-16T19:17:51.135063899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:18:02.012812655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:18:25.961103502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:18:32.015494532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:18:49.568458802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35192"
ts=2026-04-16T19:19:00.287880119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:46724"
ts=2026-04-16T19:19:02.019139386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:19:15.082125894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:19:15.086936457Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:19:32.022785932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2db89915 10.129.0.79:7946"
level=debug ts=2026-04-16T19:19:55.047341845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:20:00.295697872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:50208"
ts=2026-04-16T19:20:02.025591305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:20:19.576881117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34958"
ts=2026-04-16T19:20:20.391598394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49296"
ts=2026-04-16T19:20:32.028154682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:20:39.313047619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:20:49.580423959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:54458"
ts=2026-04-16T19:21:02.031609863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:21:19.583901494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:52480"
ts=2026-04-16T19:21:20.396306405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49922"
ts=2026-04-16T19:21:31.230916482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57726"
ts=2026-04-16T19:21:31.24132435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57738"
ts=2026-04-16T19:21:32.034860399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c57c87c9 10.129.0.79:7946"
level=debug ts=2026-04-16T19:21:38.698796325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:21:40.742561536Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2db89915 (timeout reached)"
ts=2026-04-16T19:21:43.742036756Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2db89915 has failed, no acks received"
ts=2026-04-16T19:22:00.743176667Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2db89915 (timeout reached)"
ts=2026-04-16T19:22:02.038076662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:22:03.742372372Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2db89915 has failed, no acks received"
ts=2026-04-16T19:22:32.040975212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:22:38.333601659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:23:02.043564788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:23:27.902005894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:23:27.903767224Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:23:32.04600345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:23:50.409427954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39062"
ts=2026-04-16T19:24:02.048761338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:24:12.614493848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:55254"
level=debug ts=2026-04-16T19:24:18.785643512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:24:30.330835073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:59240"
ts=2026-04-16T19:24:32.051925796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:24:42.617314364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57666"
ts=2026-04-16T19:25:00.332921205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:48034"
ts=2026-04-16T19:25:02.055226025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=error ts=2026-04-16T19:25:14.911448428Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:25:14.913953758Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:25:14.916294253Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:25:14.916334039Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-16T19:25:20.443990382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:25:32.05886834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:25:42.623808444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60814"
ts=2026-04-16T19:26:00.338371502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:42098"
ts=2026-04-16T19:26:02.076871938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c57c87c9 10.129.0.79:7946"
level=debug ts=2026-04-16T19:26:05.85956555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:26:12.626998867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:43294"
ts=2026-04-16T19:26:32.102450571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:26:42.629741957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36720"
ts=2026-04-16T19:26:50.427124712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45556"
level=debug ts=2026-04-16T19:26:56.203272588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:27:02.107418706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:27:32.110082546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c57c87c9 10.129.0.79:7946"
level=debug ts=2026-04-16T19:27:36.816521991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:27:36.818388984Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:27:42.635808266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:39154"
ts=2026-04-16T19:28:02.112955125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:28:12.638407492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:54136"
ts=2026-04-16T19:28:20.4384556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:40738"
ts=2026-04-16T19:28:32.115407592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:28:35.09816159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:29:02.118301547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c57c87c9 10.129.0.79:7946"
ts=2026-04-16T19:29:30.361034305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:51998"
level=debug ts=2026-04-16T19:29:31.604350623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:29:32.121733382Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-c57c87c9 failed: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-04-16T19:29:40.743453697Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c57c87c9 (timeout reached)"
ts=2026-04-16T19:29:43.743503899Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c57c87c9 has failed, no acks received"
ts=2026-04-16T19:29:47.057892183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34520"
ts=2026-04-16T19:29:47.068531528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34534"
ts=2026-04-16T19:29:50.743030691Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c57c87c9 (timeout reached)"
ts=2026-04-16T19:29:53.743316515Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c57c87c9 has failed, no acks received"
ts=2026-04-16T19:29:58.804066289Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c57c87c9 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T19:30:02.123065361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aa5032a6 10.129.0.79:7946"
ts=2026-04-16T19:30:20.447247997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44198"
level=debug ts=2026-04-16T19:30:20.808781291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:30:32.125857193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:30:59.895471929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:37526"
ts=2026-04-16T19:31:02.128358195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:31:18.26050591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:31:32.130896665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aa5032a6 10.129.0.79:7946"
ts=2026-04-16T19:31:50.456602268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57400"
ts=2026-04-16T19:32:02.133783173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:32:07.97147701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:32:07.9734946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:32:32.136299281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aa5032a6 10.129.0.79:7946"
ts=2026-04-16T19:33:00.38909492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:48736"
ts=2026-04-16T19:33:02.138823281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:33:09.814162014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:33:30.392056778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:45792"
ts=2026-04-16T19:33:32.1417144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aa5032a6 10.129.0.79:7946"
ts=2026-04-16T19:33:50.465761282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43344"
ts=2026-04-16T19:34:02.144295003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:34:07.235395215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:34:32.14684389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:35:02.149778634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:35:04.454935171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:35:14.913632647Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:35:14.916001344Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:35:14.918454069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:35:14.918486249Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:35:29.918003624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58022"
ts=2026-04-16T19:35:32.152497849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:35:56.332085924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:35:59.920944964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45174"
ts=2026-04-16T19:36:02.155747872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:36:20.742784752Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-aa5032a6 (timeout reached)"
ts=2026-04-16T19:36:23.742352959Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-aa5032a6 has failed, no acks received"
ts=2026-04-16T19:36:32.158998277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:36:40.743476514Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-aa5032a6 (timeout reached)"
ts=2026-04-16T19:36:42.687967767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35568"
ts=2026-04-16T19:36:42.700070809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:35576"
ts=2026-04-16T19:36:43.743303471Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-aa5032a6 has failed, no acks received"
level=debug ts=2026-04-16T19:36:48.830475752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:36:48.832433036Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:36:50.479927036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46426"
ts=2026-04-16T19:37:00.416043977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:60410"
ts=2026-04-16T19:37:02.161711148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:37:30.418349081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:49746"
ts=2026-04-16T19:37:32.164291123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:37:37.72211017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:38:02.167735247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4b9df961 10.129.0.79:7946"
level=debug ts=2026-04-16T19:38:17.121451602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:38:32.17026743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:38:50.493200676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55016"
ts=2026-04-16T19:38:52.473796514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46082"
ts=2026-04-16T19:39:02.172868223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:39:12.702664674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:39:30.430781714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:59024"
ts=2026-04-16T19:39:32.176471481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:40:02.179930644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
level=debug ts=2026-04-16T19:40:02.952244583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:40:22.482133772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42252"
ts=2026-04-16T19:40:32.182720658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-4b9df961 10.129.0.79:7946"
level=debug ts=2026-04-16T19:40:49.64630847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:40:49.648165156Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:40:50.505501817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54504"
ts=2026-04-16T19:41:02.186915696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:41:30.442601352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:35970"
ts=2026-04-16T19:41:32.189929306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:41:32.776197956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:42:00.743450036Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4b9df961 (timeout reached)"
ts=2026-04-16T19:42:02.191983932Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-4b9df961 failed: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-04-16T19:42:03.743016872Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4b9df961 has failed, no acks received"
level=debug ts=2026-04-16T19:42:17.820807588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:42:20.743027515Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-4b9df961 (timeout reached)"
ts=2026-04-16T19:42:23.743821807Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-4b9df961 has failed, no acks received"
ts=2026-04-16T19:42:23.743938438Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-4b9df961 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T19:42:32.193885732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:42:34.684031318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38504"
ts=2026-04-16T19:42:34.695411552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38514"
ts=2026-04-16T19:42:50.515914282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49566"
ts=2026-04-16T19:43:00.450432572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:58328"
ts=2026-04-16T19:43:02.196632913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9a858177 10.129.0.79:7946"
level=debug ts=2026-04-16T19:43:13.18063353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:43:32.199411701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:43:48.933899949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:43:50.521277014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42786"
ts=2026-04-16T19:44:00.743238868Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a858177 (timeout reached)"
ts=2026-04-16T19:44:02.201222882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:44:03.743416594Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a858177 has failed, no acks received"
ts=2026-04-16T19:44:10.74272239Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a858177 (timeout reached)"
ts=2026-04-16T19:44:13.742545803Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a858177 has failed, no acks received"
ts=2026-04-16T19:44:20.524447059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39990"
ts=2026-04-16T19:44:20.742544782Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-9a858177 (timeout reached)"
ts=2026-04-16T19:44:23.742305959Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-9a858177 has failed, no acks received"
ts=2026-04-16T19:44:23.743508054Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-9a858177 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T19:44:30.460065982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:60294"
ts=2026-04-16T19:44:32.20316376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:44:39.042548254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:44:39.04452367Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:45:02.205899851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=error ts=2026-04-16T19:45:14.911723151Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:45:14.913896795Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:45:14.915634459Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:45:14.915658385Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:45:19.653261199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42460"
ts=2026-04-16T19:45:19.664120678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42462"
ts=2026-04-16T19:45:20.530550769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38048"
ts=2026-04-16T19:45:32.208067452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:45:34.561216927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:46:02.21069891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
level=debug ts=2026-04-16T19:46:12.431346549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:46:15.742614193Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5c4e24c3 (timeout reached)"
ts=2026-04-16T19:46:18.743021251Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5c4e24c3 has failed, no acks received"
ts=2026-04-16T19:46:20.535408746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33020"
ts=2026-04-16T19:46:30.471651727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:37102"
ts=2026-04-16T19:46:30.742938985Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5c4e24c3 (timeout reached)"
ts=2026-04-16T19:46:32.213569063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-pcx2j-daadf390 10.129.0.78:7946"
ts=2026-04-16T19:46:33.743390667Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5c4e24c3 has failed, no acks received"
ts=2026-04-16T19:46:35.743653092Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5c4e24c3 (timeout reached)"
ts=2026-04-16T19:46:38.743333696Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-5c4e24c3 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-16T19:46:38.744404985Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5c4e24c3 has failed, no acks received"
level=debug ts=2026-04-16T19:46:52.122410842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:47:00.474243705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:48834"
ts=2026-04-16T19:47:02.216364921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"
ts=2026-04-16T19:47:30.476834009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.53:54236"
ts=2026-04-16T19:47:32.218568596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8cv4w-044c1587 10.128.0.53:7946"