level=info ts=2026-04-28T09:34:39.363998989Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-28T09:34:39.364046305Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-28T09:34:39.366016668Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T09:34:39.366477932Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:34:39.366602728Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=debug ts=2026-04-28T09:34:39.367196625Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:34:39.367278429Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=info ts=2026-04-28T09:34:39.367325303Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-7cfd2332
ts=2026-04-28T09:34:39.367384598Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T09:34:39.367396258Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=debug ts=2026-04-28T09:34:39.367409689Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=info ts=2026-04-28T09:34:39.367507703Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T09:34:39.375244669Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T09:34:39.37526454Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T09:34:39.375276169Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.770538ms
level=error ts=2026-04-28T09:34:39.375286676Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-28T09:34:39.375300011Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-28T09:34:39.377868467Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:34:39.377885977Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T09:34:39.382146615Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-28T09:34:39.382155718Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-28T09:34:39.38216699Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-28T09:34:39.382168621Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-28T09:34:39.382190375Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-28T09:34:39.382266738Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-28T09:34:39.382286203Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T09:34:39.382312835Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-28T09:34:39.382434113Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-28T09:34:39.382440234Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-28T09:34:39.38248232Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-28T09:34:39.382491079Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-28T09:34:39.382517304Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T09:34:39.382663961Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-28T09:34:39.48809783Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-28T09:34:39.488145676Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-28T09:34:40.488715346Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:34:40.634108839Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:34:40.634160508Z caller=loki.go:599 msg="Loki started" startup_time=1.333944361s
ts=2026-04-28T09:34:40.674766051Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:34:40.674798396Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:34:43.125971844Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:34:43.125994559Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T09:34:45.634744206Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-28T09:34:45.634821532Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-28T09:34:48.075735699Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:34:48.075758963Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T09:34:56.058126769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:34:58.353598848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44160"
ts=2026-04-28T09:34:59.736739649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.78:7946"
ts=2026-04-28T09:34:59.736863287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:43382"
ts=2026-04-28T09:34:59.738060811Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.80:7946: dial tcp 10.130.0.80:7946: connect: connection refused"
level=info ts=2026-04-28T09:34:59.738074235Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.362767754s
ts=2026-04-28T09:35:03.674479285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47954"
ts=2026-04-28T09:35:06.199884439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:32900"
level=debug ts=2026-04-28T09:35:10.472168111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:35:26.843957113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39658"
ts=2026-04-28T09:35:27.659203953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:35:29.323914909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:35:36.195827309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58548"
level=debug ts=2026-04-28T09:35:53.898852581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:35:57.661579612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:36:06.198864429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53540"
level=debug ts=2026-04-28T09:36:19.145640526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T09:36:19.147546843Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:36:27.663634302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:36:57.665253938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:37:06.098645606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:37:27.666302228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:37:36.204882261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60076"
ts=2026-04-28T09:37:42.123826582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51032"
ts=2026-04-28T09:37:57.668278435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:37:58.646939541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:38:12.127285911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60120"
ts=2026-04-28T09:38:27.670251007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:38:54.853927659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:38:57.672170438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:39:27.675423789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:39:56.869397788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55554"
ts=2026-04-28T09:39:57.678613388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:40:03.293600787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:40:06.215703189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39946"
ts=2026-04-28T09:40:27.681188359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:40:42.141105098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40120"
ts=2026-04-28T09:40:57.683804064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:41:11.12899146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T09:41:11.130655765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:41:26.876794911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60850"
ts=2026-04-28T09:41:27.686849492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:41:36.223069884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57346"
level=debug ts=2026-04-28T09:41:48.95095388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:41:57.690479165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:42:06.226396321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59082"
ts=2026-04-28T09:42:12.146801885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52406"
ts=2026-04-28T09:42:26.881279705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41076"
ts=2026-04-28T09:42:27.69302445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:42:36.229331511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55174"
ts=2026-04-28T09:42:42.14940592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41910"
level=debug ts=2026-04-28T09:42:42.169138515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:42:57.695038479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:43:06.232000821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34320"
ts=2026-04-28T09:43:12.152966424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52318"
ts=2026-04-28T09:43:26.885858376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44510"
ts=2026-04-28T09:43:27.69728361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:43:35.663562955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:43:36.235562457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40218"
ts=2026-04-28T09:43:42.156052632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37638"
ts=2026-04-28T09:43:57.699314691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:44:06.238951629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50650"
level=debug ts=2026-04-28T09:44:25.330019308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:44:26.88944743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58920"
ts=2026-04-28T09:44:27.702002936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:44:36.241839958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46578"
level=info ts=2026-04-28T09:44:45.635799051Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-28T09:44:45.64184212Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:44:45.643991804Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:44:45.645861469Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:44:45.645892317Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T09:44:45.645904438Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-28T09:44:57.704177984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:45:24.034403066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T09:45:24.036124478Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:45:27.706360098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:45:57.709168342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:46:11.209767414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:46:27.711718745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:46:56.904180627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48056"
ts=2026-04-28T09:46:57.713855664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:47:13.255187596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:47:26.905547054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38810"
ts=2026-04-28T09:47:27.716682382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:47:50.21446008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:47:57.719514258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:48:27.721531539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:48:37.899359392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:48:57.723675318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:49:26.088179789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T09:49:26.09006095Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:49:27.726224399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:49:56.920044446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58484"
ts=2026-04-28T09:49:57.729071076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:50:12.185110596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56682"
level=debug ts=2026-04-28T09:50:20.403423176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:50:27.731107681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:50:57.732884275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:50:57.915507232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:51:26.928255976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46880"
ts=2026-04-28T09:51:27.735629918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:51:36.632569962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:51:42.189912946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58062"
ts=2026-04-28T09:51:56.93037195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43612"
ts=2026-04-28T09:51:57.739083959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:52:26.931876101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54350"
ts=2026-04-28T09:52:27.741948963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:52:34.654239296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:52:36.275487882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43492"
ts=2026-04-28T09:52:57.744988943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:53:12.197685246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42666"
ts=2026-04-28T09:53:26.937058367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52114"
ts=2026-04-28T09:53:27.747380689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:53:34.611148062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T09:53:34.613272751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:53:36.282165123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53576"
ts=2026-04-28T09:53:57.748657769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T09:54:13.760071845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:54:27.750457946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=error ts=2026-04-28T09:54:45.651553635Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:54:45.653317944Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:54:45.65506421Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T09:54:45.655096255Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:54:57.753207839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:55:00.180146804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:55:12.211108795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35714"
ts=2026-04-28T09:55:27.756402048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:55:38.263065122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:55:42.214263353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:32934"
ts=2026-04-28T09:55:57.759621375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T09:56:15.420065475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:56:27.761747852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:56:42.219208838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43362"
ts=2026-04-28T09:56:57.762831855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:57:12.222591949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57036"
level=debug ts=2026-04-28T09:57:16.07889791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T09:57:16.08103725Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T09:57:26.962442873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53088"
ts=2026-04-28T09:57:27.764682411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T09:57:36.300474656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55092"
ts=2026-04-28T09:57:57.76751356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:58:08.250021797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:58:26.966468231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54788"
ts=2026-04-28T09:58:27.770589606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T09:58:36.307621459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36034"
ts=2026-04-28T09:58:57.773468826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T09:59:06.246218431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T09:59:12.232766653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:33554"
ts=2026-04-28T09:59:27.775752016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T09:59:42.235875339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58912"
ts=2026-04-28T09:59:57.777851229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:00:00.165625929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:00:06.316730747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45730"
ts=2026-04-28T10:00:12.239261507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56246"
ts=2026-04-28T10:00:26.977153408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40404"
ts=2026-04-28T10:00:27.78003504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T10:00:36.31934435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45464"
level=debug ts=2026-04-28T10:00:49.597771687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:00:56.978657878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36570"
ts=2026-04-28T10:00:57.786920401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:01:27.790501759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:01:34.616338955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:01:34.618157559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:01:57.793328074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:02:12.249848535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48818"
ts=2026-04-28T10:02:27.79722709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:02:29.089093737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:02:36.329168201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34300"
ts=2026-04-28T10:02:42.252614904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39696"
ts=2026-04-28T10:02:57.799471579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T10:03:05.975819045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:03:27.80190319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:03:41.229839787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:03:57.806698057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:04:12.259095066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50742"
level=debug ts=2026-04-28T10:04:23.548381238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:04:27.809760696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=error ts=2026-04-28T10:04:45.658628233Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:04:45.661151149Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:04:45.663237598Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:04:45.66327187Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:04:57.007095265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47092"
ts=2026-04-28T10:04:57.813155959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:05:11.669579888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:05:11.671277744Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:05:27.815589598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
level=debug ts=2026-04-28T10:05:56.806889289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:05:57.019423829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57926"
ts=2026-04-28T10:05:57.81802403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:06:27.820028878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fe2c2b8c 10.130.0.79:7946"
ts=2026-04-28T10:06:42.272073532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51802"
level=debug ts=2026-04-28T10:06:50.960969683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:06:57.024232428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37140"
ts=2026-04-28T10:06:57.822117936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:07:08.024531225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48050"
ts=2026-04-28T10:07:08.038479829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48060"
ts=2026-04-28T10:07:11.369359758Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-fe2c2b8c (timeout reached)"
ts=2026-04-28T10:07:14.369424603Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-fe2c2b8c has failed, no acks received"
ts=2026-04-28T10:07:27.824572397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
ts=2026-04-28T10:07:31.368358938Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-fe2c2b8c (timeout reached)"
ts=2026-04-28T10:07:34.369209579Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-fe2c2b8c has failed, no acks received"
ts=2026-04-28T10:07:34.370288008Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-fe2c2b8c as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-28T10:07:52.157085212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:07:57.826039979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
ts=2026-04-28T10:08:27.828425731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:08:46.443254379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52504"
level=debug ts=2026-04-28T10:08:50.823501448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:08:57.831202114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:09:06.363747625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50824"
ts=2026-04-28T10:09:12.290023983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45864"
ts=2026-04-28T10:09:27.832864024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:09:31.157292049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:09:31.159997758Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:09:36.367030439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33896"
ts=2026-04-28T10:09:57.835509324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:10:23.62065229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:10:27.838066788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
ts=2026-04-28T10:10:57.839426606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:11:17.779382734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:11:27.84223137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:11:36.380876874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50040"
ts=2026-04-28T10:11:42.305475717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50986"
ts=2026-04-28T10:11:57.844894282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:12:13.958542388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:12:27.845725677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:12:49.708006412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:12:57.847608308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:13:06.389318956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41376"
ts=2026-04-28T10:13:27.850362913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:13:46.474175607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60488"
level=debug ts=2026-04-28T10:13:48.144193595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:13:48.146037756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:13:57.852500207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
ts=2026-04-28T10:14:27.854809329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:14:40.930094604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:14:45.651052595Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:14:45.652929876Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:14:45.654726202Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:14:45.654770853Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:14:57.856951498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
ts=2026-04-28T10:15:06.405740893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41974"
ts=2026-04-28T10:15:27.859106172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:15:33.072179757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:15:36.418749783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54222"
ts=2026-04-28T10:15:57.861883404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:16:11.101275529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:27.863454581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:16:45.655845306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:57.865376921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:17:06.425941076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34058"
ts=2026-04-28T10:17:16.503235826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44746"
ts=2026-04-28T10:17:26.470280583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49106"
ts=2026-04-28T10:17:26.482826104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49120"
ts=2026-04-28T10:17:27.86782812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-138c0432 10.130.0.79:7946"
level=debug ts=2026-04-28T10:17:47.520804792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:17:47.522790356Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:17:51.368903438Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-138c0432 (timeout reached)"
ts=2026-04-28T10:17:54.368325367Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-138c0432 has failed, no acks received"
ts=2026-04-28T10:17:57.869591598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:18:03.173833852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45068"
ts=2026-04-28T10:18:06.429821648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59058"
ts=2026-04-28T10:18:27.873028316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:18:36.43295438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38984"
level=debug ts=2026-04-28T10:18:46.659913825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:18:57.875278653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b1fc7580 10.130.0.79:7946"
ts=2026-04-28T10:19:12.340982434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46238"
ts=2026-04-28T10:19:27.876548244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b1fc7580 10.130.0.79:7946"
ts=2026-04-28T10:19:33.182029803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54598"
level=debug ts=2026-04-28T10:19:44.347371583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:19:57.878830625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:20:06.441081889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48136"
ts=2026-04-28T10:20:27.881746371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:20:33.186916178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33508"
level=debug ts=2026-04-28T10:20:40.402963314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:20:42.346061049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34652"
ts=2026-04-28T10:20:57.885311639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:21:06.445232048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53322"
ts=2026-04-28T10:21:27.888153744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:21:30.358052318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:21:36.448427582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42932"
ts=2026-04-28T10:21:57.891416883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:22:03.195863271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55214"
ts=2026-04-28T10:22:06.451298569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47242"
level=debug ts=2026-04-28T10:22:17.116616762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:22:17.119143836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:22:27.894871278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:22:42.359143856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:54474"
level=debug ts=2026-04-28T10:22:52.661760954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:22:57.897232603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b1fc7580 10.130.0.79:7946"
ts=2026-04-28T10:23:06.45781274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53594"
ts=2026-04-28T10:23:27.899769677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:23:33.20429184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43320"
ts=2026-04-28T10:23:36.461322758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60600"
level=debug ts=2026-04-28T10:23:36.942155921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:23:57.903432553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:24:06.464424651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36118"
ts=2026-04-28T10:24:27.90652537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:24:30.046273532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:24:36.467850603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58366"
level=error ts=2026-04-28T10:24:45.649432479Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:24:45.651268734Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:24:45.653222353Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:24:45.653247252Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:24:46.368896086Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b1fc7580 (timeout reached)"
ts=2026-04-28T10:24:46.532179445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56474"
ts=2026-04-28T10:24:46.545743854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56476"
ts=2026-04-28T10:24:49.368531956Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b1fc7580 has failed, no acks received"
ts=2026-04-28T10:24:57.909923324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:25:04.970564999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:25:06.368272107Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b1fc7580 (timeout reached)"
ts=2026-04-28T10:25:09.368168271Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b1fc7580 has failed, no acks received"
ts=2026-04-28T10:25:27.92324462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e475354b 10.130.0.79:7946"
level=debug ts=2026-04-28T10:25:43.448959658Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:25:43.45114419Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:25:56.92749918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37484"
ts=2026-04-28T10:25:57.925426027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:26:12.373831713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55406"
ts=2026-04-28T10:26:26.928365636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39202"
ts=2026-04-28T10:26:27.928679831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:26:32.13784791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:26:57.931193648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:27:06.48202947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44066"
ts=2026-04-28T10:27:12.380723078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60412"
level=debug ts=2026-04-28T10:27:25.569187788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:27:26.933058763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34888"
ts=2026-04-28T10:27:27.933450012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e475354b 10.130.0.79:7946"
ts=2026-04-28T10:27:36.485331163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47132"
ts=2026-04-28T10:27:57.934737752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e475354b 10.130.0.79:7946"
ts=2026-04-28T10:28:06.488238056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39414"
level=debug ts=2026-04-28T10:28:25.319901962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:28:27.937247282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:28:36.491598601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49254"
ts=2026-04-28T10:28:42.389364239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34534"
ts=2026-04-28T10:28:56.94031562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57192"
ts=2026-04-28T10:28:57.940389184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:29:06.495381079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47142"
level=debug ts=2026-04-28T10:29:26.06406681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:29:27.942631247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e475354b 10.130.0.79:7946"
ts=2026-04-28T10:29:57.944652397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:30:00.744974683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:30:00.746746608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:30:12.397956404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44420"
ts=2026-04-28T10:30:27.946229625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e475354b 10.130.0.79:7946"
level=debug ts=2026-04-28T10:30:50.108114192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:30:57.948248696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:31:06.368654519Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e475354b (timeout reached)"
ts=2026-04-28T10:31:09.368821812Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e475354b has failed, no acks received"
ts=2026-04-28T10:31:21.535223061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43274"
ts=2026-04-28T10:31:21.548426622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43286"
ts=2026-04-28T10:31:27.951694764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:31:35.353709488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:31:36.509865994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51430"
ts=2026-04-28T10:31:57.954927459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:32:15.682302279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:32:27.957570253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f48f6f2 10.130.0.79:7946"
ts=2026-04-28T10:32:57.959371497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f48f6f2 10.130.0.79:7946"
level=debug ts=2026-04-28T10:33:14.135277236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:33:27.961620939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:33:35.922002279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38596"
ts=2026-04-28T10:33:36.523872118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49402"
level=debug ts=2026-04-28T10:33:52.317149012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:33:52.318827254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:33:57.964669564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:34:05.923242279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54246"
ts=2026-04-28T10:34:27.968286852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:34:40.173424751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:34:45.650662202Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:34:45.652497381Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:34:45.65429785Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:34:45.654324363Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:34:57.971392121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:35:05.928120927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48326"
ts=2026-04-28T10:35:12.434465136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37580"
level=debug ts=2026-04-28T10:35:23.750674884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:35:27.975038972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:35:42.437309912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38968"
ts=2026-04-28T10:35:57.977491014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f48f6f2 10.130.0.79:7946"
level=debug ts=2026-04-28T10:36:10.193196664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:36:27.979196557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-3f48f6f2 10.130.0.79:7946"
ts=2026-04-28T10:36:57.98190699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:36:59.029769935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:37:21.368915688Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3f48f6f2 (timeout reached)"
ts=2026-04-28T10:37:24.36867688Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3f48f6f2 has failed, no acks received"
ts=2026-04-28T10:37:26.36902582Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-3f48f6f2 (timeout reached)"
ts=2026-04-28T10:37:27.985095371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:37:29.369575404Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-3f48f6f2 has failed, no acks received"
ts=2026-04-28T10:37:36.542971543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38482"
ts=2026-04-28T10:37:42.444509942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44736"
level=debug ts=2026-04-28T10:37:48.188046841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:37:48.189963366Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:37:57.988020985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:38:00.517478834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52976"
ts=2026-04-28T10:38:00.529814343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52992"
ts=2026-04-28T10:38:06.546660368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36278"
ts=2026-04-28T10:38:27.990423437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
level=debug ts=2026-04-28T10:38:28.236662948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:38:42.449875593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56004"
ts=2026-04-28T10:38:57.478543141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60686"
ts=2026-04-28T10:38:57.992470788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:39:12.253170597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:39:27.99862822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:39:42.456422573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53286"
ts=2026-04-28T10:39:58.001499764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:40:08.555200196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:28.004123938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
level=debug ts=2026-04-28T10:40:54.696017752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:58.005229357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
ts=2026-04-28T10:41:06.555545488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41898"
ts=2026-04-28T10:41:12.461757791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56330"
ts=2026-04-28T10:41:28.007886075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:41:29.054799538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:41:29.056541247Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:41:36.558124861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54822"
ts=2026-04-28T10:41:42.46475405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50238"
ts=2026-04-28T10:41:58.012210244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:42:06.561576688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35748"
ts=2026-04-28T10:42:27.498485387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49534"
ts=2026-04-28T10:42:28.015687803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:42:28.508859176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:42:36.565014306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40812"
ts=2026-04-28T10:42:58.018555219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:43:06.567902004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49896"
level=debug ts=2026-04-28T10:43:09.614042516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:43:28.02035685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
ts=2026-04-28T10:43:36.570263388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51088"
ts=2026-04-28T10:43:57.505647418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:32794"
ts=2026-04-28T10:43:58.021716064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
level=debug ts=2026-04-28T10:44:06.941276647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:44:28.023834073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:44:36.576619841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37504"
level=error ts=2026-04-28T10:44:45.649647136Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:44:45.651694972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:44:45.653498858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:44:45.653528456Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T10:44:49.82223275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:44:58.025641372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
ts=2026-04-28T10:45:06.579901291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54078"
level=debug ts=2026-04-28T10:45:24.890914684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:45:24.892595598Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:45:27.513443079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43834"
ts=2026-04-28T10:45:28.027455071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e45ffd 10.130.0.79:7946"
ts=2026-04-28T10:45:57.514370385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53224"
ts=2026-04-28T10:45:58.02924772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
level=debug ts=2026-04-28T10:46:25.781010463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:46:28.032179579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:46:36.368280904Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-04e45ffd (timeout reached)"
ts=2026-04-28T10:46:36.584390255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37474"
ts=2026-04-28T10:46:39.367953917Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-04e45ffd has failed, no acks received"
ts=2026-04-28T10:46:46.553084123Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-04e45ffd as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:46:58.035035942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:47:06.587588002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38820"
ts=2026-04-28T10:47:12.489631347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51866"
level=debug ts=2026-04-28T10:47:18.298900912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:47:28.038354556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:47:36.590495647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42846"
ts=2026-04-28T10:47:43.50540839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35836"
ts=2026-04-28T10:47:43.517461637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35852"
ts=2026-04-28T10:47:58.040939442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
level=debug ts=2026-04-28T10:48:03.007034647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:48:28.042856462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:48:36.594658845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46916"
ts=2026-04-28T10:48:40.027337013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55590"
level=debug ts=2026-04-28T10:48:40.38367812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:48:58.045118963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
level=debug ts=2026-04-28T10:49:21.782756633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:49:21.784728473Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:49:28.047121793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4st9q-235205aa 10.129.0.70:7946"
ts=2026-04-28T10:49:58.049341242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
ts=2026-04-28T10:50:06.604600692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54114"
ts=2026-04-28T10:50:12.502714585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59766"
level=debug ts=2026-04-28T10:50:18.812397274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:50:28.050417874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
ts=2026-04-28T10:50:40.037870202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48392"
ts=2026-04-28T10:50:42.505863724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49880"
ts=2026-04-28T10:50:58.05125556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
ts=2026-04-28T10:51:06.609963745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39534"
ts=2026-04-28T10:51:10.039679784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45180"
ts=2026-04-28T10:51:12.509314618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:32860"
level=debug ts=2026-04-28T10:51:13.139235401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:51:28.053208364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
ts=2026-04-28T10:51:58.056164931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-dlxrb-120f522c 10.129.0.69:7946"
level=debug ts=2026-04-28T10:52:12.848530046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:52:28.05847668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
ts=2026-04-28T10:52:36.617578705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45690"
level=debug ts=2026-04-28T10:52:49.766458775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:52:58.060331662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ad23ed26 10.130.0.79:7946"
ts=2026-04-28T10:53:06.621104097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33092"