=== Cluster state 2026-07-15T21:49:07+00:00 === NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME ip-10-0-131-113.ec2.internal Ready worker 39m v1.34.9 10.0.131.113 Red Hat Enterprise Linux CoreOS 9.6.20260702-0 (Plow) 5.14.0-570.126.1.el9_6.x86_64 cri-o://1.34.10-2.rhaos4.21.git83ee25b.el9 ip-10-0-136-220.ec2.internal Ready worker 39m v1.34.9 10.0.136.220 Red Hat Enterprise Linux CoreOS 9.6.20260702-0 (Plow) 5.14.0-570.126.1.el9_6.x86_64 cri-o://1.34.10-2.rhaos4.21.git83ee25b.el9 ip-10-0-143-128.ec2.internal Ready worker 39m v1.34.9 10.0.143.128 Red Hat Enterprise Linux CoreOS 9.6.20260702-0 (Plow) 5.14.0-570.126.1.el9_6.x86_64 cri-o://1.34.10-2.rhaos4.21.git83ee25b.el9 NAME STATUS AGE ai-tenants Active 29m cert-manager Active 34m cert-manager-operator Active 34m default Active 47m kuadrant-system Active 31m kube-node-lease Active 47m kube-public Active 47m kube-system Active 47m llm Active 28m maas-admin Active 26m models-as-a-service Active 29m odh-ai-gateway-infra Active 29m open-cluster-management-0b31ce17-7905-4df7-813a-813829341 Active 44m open-cluster-management-agent-addon Active 44m opendatahub Active 33m opendatahub-monitoring Active 33m openshift Active 46m openshift-apiserver Active 46m openshift-apiserver-operator Active 46m openshift-authentication Active 46m openshift-authentication-operator Active 46m openshift-cloud-controller-manager Active 46m openshift-cloud-credential-operator Active 46m openshift-cloud-network-config-controller Active 46m openshift-cluster-csi-drivers Active 46m openshift-cluster-machine-approver Active 46m openshift-cluster-node-tuning-operator Active 46m openshift-cluster-samples-operator Active 46m openshift-cluster-storage-operator Active 46m openshift-cluster-version Active 46m openshift-config Active 46m openshift-config-managed Active 46m openshift-config-operator Active 46m openshift-console Active 46m openshift-console-operator Active 46m openshift-console-user-settings Active 46m openshift-controller-manager Active 46m openshift-controller-manager-operator Active 46m openshift-dns Active 46m openshift-dns-operator Active 46m openshift-etcd Active 46m openshift-host-network Active 45m openshift-image-registry Active 46m openshift-infra Active 46m openshift-ingress Active 46m openshift-ingress-canary Active 46m openshift-ingress-operator Active 46m openshift-insights Active 46m openshift-kube-apiserver Active 46m openshift-kube-apiserver-operator Active 46m openshift-kube-controller-manager Active 46m openshift-kube-controller-manager-operator Active 46m openshift-kube-scheduler Active 46m openshift-kube-scheduler-operator Active 46m openshift-kube-storage-version-migrator Active 38m openshift-kube-storage-version-migrator-operator Active 46m openshift-lws-operator Active 34m openshift-machine-api Active 46m openshift-machine-config-operator Active 46m openshift-marketplace Active 46m openshift-monitoring Active 46m openshift-multus Active 45m openshift-network-console Active 45m openshift-network-diagnostics Active 45m openshift-network-node-identity Active 45m openshift-network-operator Active 46m openshift-node Active 46m openshift-operator-lifecycle-manager Active 46m openshift-operators Active 46m openshift-ovn-kubernetes Active 45m openshift-route-controller-manager Active 46m openshift-service-ca Active 38m openshift-service-ca-operator Active 46m openshift-user-workload-monitoring Active 46m premium-users-namespace Active 26m --- MaaS controller namespace (opendatahub) --- NAME READY STATUS RESTARTS AGE pod/kserve-controller-manager-d6f4b65b8-76phf 1/1 Running 0 32m pod/llmisvc-controller-manager-777c6bd7d5-plf56 1/1 Running 0 32m pod/maas-controller-d79b5d84b-h7tmp 1/1 Running 0 25m pod/model-serving-api-54d89bd5fc-69s79 1/1 Running 0 32m pod/odh-model-controller-7998c67dc4-x7ftv 1/1 Running 0 32m pod/opendatahub-operator-controller-manager-7d5d77c45b-6zkfh 1/1 Running 0 33m pod/opendatahub-operator-controller-manager-7d5d77c45b-9986p 1/1 Running 0 33m pod/opendatahub-operator-controller-manager-7d5d77c45b-fr6pn 1/1 Running 0 33m NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/kserve-controller-manager-metrics-service ClusterIP 172.31.18.179 8443/TCP 32m service/kserve-controller-manager-service ClusterIP 172.31.167.64 8443/TCP 32m service/kserve-webhook-server-service ClusterIP 172.31.138.103 443/TCP 32m service/llmisvc-controller-manager-service ClusterIP 172.31.2.14 8443/TCP 32m service/llmisvc-webhook-server-service ClusterIP 172.31.239.141 443/TCP 32m service/maas-controller-webhook-service ClusterIP 172.31.15.141 443/TCP 29m service/model-serving-api ClusterIP 172.31.229.26 443/TCP,8080/TCP 32m service/odh-model-controller-metrics-service ClusterIP 172.31.90.75 8080/TCP 32m service/odh-model-controller-webhook-service ClusterIP 172.31.218.137 443/TCP 32m service/opendatahub-operator-controller-manager-metrics-service ClusterIP 172.31.142.14 8443/TCP 33m service/opendatahub-operator-controller-manager-service ClusterIP 172.31.171.52 443/TCP 33m NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/kserve-controller-manager 1/1 1 1 32m deployment.apps/llmisvc-controller-manager 1/1 1 1 32m deployment.apps/maas-controller 1/1 1 1 29m deployment.apps/model-serving-api 1/1 1 1 32m deployment.apps/odh-model-controller 1/1 1 1 32m deployment.apps/opendatahub-operator-controller-manager 3/3 3 3 33m NAME DESIRED CURRENT READY AGE replicaset.apps/kserve-controller-manager-d6f4b65b8 1 1 1 32m replicaset.apps/llmisvc-controller-manager-777c6bd7d5 1 1 1 32m replicaset.apps/maas-controller-bdd575fbb 0 0 0 29m replicaset.apps/maas-controller-d79b5d84b 1 1 1 29m replicaset.apps/model-serving-api-54d89bd5fc 1 1 1 32m replicaset.apps/odh-model-controller-7998c67dc4 1 1 1 32m replicaset.apps/opendatahub-operator-controller-manager-7d5d77c45b 3 3 3 33m --- MaaS API deployment namespace (odh-ai-gateway-infra) --- NAME READY STATUS RESTARTS AGE pod/maas-api-6fd778c595-klfg6 1/1 Running 0 29m pod/maas-api-e2e-ait-d8b57005-6996ff7477-75zdh 1/1 Running 0 13m pod/maas-api-e2e-api-a-599e95-67c64789c8-24brf 1/1 Running 0 7m39s pod/maas-api-e2e-api-b-d3a40f-d8b5c8677-9g8jf 1/1 Running 0 7m31s pod/maas-api-e2e-derive-86b85e98-67f6f6f65d-tlhr2 1/1 Running 0 13m pod/maas-api-e2e-inf-a-4ec9ec-67d85575c8-96ntd 1/1 Running 0 9m39s pod/maas-api-e2e-inf-b-a5309f-d84f8d4cb-4wsl5 1/1 Running 0 9m31s pod/maas-api-e2e-migrate-620eb2f1-5f479cff7c-tl77j 1/1 Running 0 13m pod/maas-api-e2e-mt-c2c3a008-85947d79c7-2r4xq 1/1 Running 0 11m pod/maas-api-e2e-shared-a-c664d0-7fc78b5458-jxtkv 1/1 Running 0 6m49s pod/maas-api-e2e-shared-b-700cbf-654f99459b-77cgv 1/1 Running 0 6m41s pod/maas-api-key-cleanup-29735850-qnjc5 0/1 Completed 0 19m pod/maas-api-key-cleanup-29735865-tc5ds 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-ait-d8b57005-29735865-xpdbf 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-api-a-599e95-29735865-q9lbd 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-api-b-d3a40f-29735865-mgtbr 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-derive-86b85e98-29735865-v872x 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-inf-a-4ec9ec-29735865-rgnbz 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-inf-b-a5309f-29735865-frzz7 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-migrate-620eb2f1-29735865-xtt8n 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-mt-c2c3a008-29735865-pvdw5 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-shared-a-c664d0-29735865-2nbfn 0/1 Completed 0 4m8s pod/maas-api-key-cleanup-e2e-shared-b-700cbf-29735865-t28lv 0/1 Completed 0 4m8s pod/postgres-69fb4594cf-f2x8l 1/1 Running 0 29m NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/maas-api ClusterIP 172.31.219.215 8443/TCP 29m service/maas-api-e2e-ait-d8b57005 ClusterIP 172.31.212.194 8443/TCP 13m service/maas-api-e2e-api-a-599e95 ClusterIP 172.31.121.228 8443/TCP 7m39s service/maas-api-e2e-api-b-d3a40f ClusterIP 172.31.225.247 8443/TCP 7m31s service/maas-api-e2e-derive-86b85e98 ClusterIP 172.31.17.106 8443/TCP 13m service/maas-api-e2e-inf-a-4ec9ec ClusterIP 172.31.239.196 8443/TCP 9m39s service/maas-api-e2e-inf-b-a5309f ClusterIP 172.31.76.22 8443/TCP 9m31s service/maas-api-e2e-migrate-620eb2f1 ClusterIP 172.31.124.16 8443/TCP 13m service/maas-api-e2e-mt-c2c3a008 ClusterIP 172.31.241.237 8443/TCP 11m service/maas-api-e2e-shared-a-c664d0 ClusterIP 172.31.11.107 8443/TCP 6m49s service/maas-api-e2e-shared-b-700cbf ClusterIP 172.31.91.124 8443/TCP 6m41s service/postgres ClusterIP 172.31.52.224 5432/TCP 29m NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/maas-api 1/1 1 1 29m deployment.apps/maas-api-e2e-ait-d8b57005 1/1 1 1 13m deployment.apps/maas-api-e2e-api-a-599e95 1/1 1 1 7m39s deployment.apps/maas-api-e2e-api-b-d3a40f 1/1 1 1 7m31s deployment.apps/maas-api-e2e-derive-86b85e98 1/1 1 1 13m deployment.apps/maas-api-e2e-inf-a-4ec9ec 1/1 1 1 9m40s deployment.apps/maas-api-e2e-inf-b-a5309f 1/1 1 1 9m31s deployment.apps/maas-api-e2e-migrate-620eb2f1 1/1 1 1 13m deployment.apps/maas-api-e2e-mt-c2c3a008 1/1 1 1 11m deployment.apps/maas-api-e2e-shared-a-c664d0 1/1 1 1 6m49s deployment.apps/maas-api-e2e-shared-b-700cbf 1/1 1 1 6m41s deployment.apps/postgres 1/1 1 1 29m NAME DESIRED CURRENT READY AGE replicaset.apps/maas-api-6fd778c595 1 1 1 29m replicaset.apps/maas-api-e2e-ait-d8b57005-6996ff7477 1 1 1 13m replicaset.apps/maas-api-e2e-api-a-599e95-67c64789c8 1 1 1 7m39s replicaset.apps/maas-api-e2e-api-b-d3a40f-d8b5c8677 1 1 1 7m31s replicaset.apps/maas-api-e2e-derive-86b85e98-67f6f6f65d 1 1 1 13m replicaset.apps/maas-api-e2e-inf-a-4ec9ec-67d85575c8 1 1 1 9m39s replicaset.apps/maas-api-e2e-inf-b-a5309f-d84f8d4cb 1 1 1 9m31s replicaset.apps/maas-api-e2e-migrate-620eb2f1-5f479cff7c 1 1 1 13m replicaset.apps/maas-api-e2e-mt-c2c3a008-85947d79c7 1 1 1 11m replicaset.apps/maas-api-e2e-shared-a-c664d0-7fc78b5458 1 1 1 6m49s replicaset.apps/maas-api-e2e-shared-b-700cbf-654f99459b 1 1 1 6m41s replicaset.apps/postgres-69fb4594cf 1 1 1 29m NAME SCHEDULE TIMEZONE SUSPEND ACTIVE LAST SCHEDULE AGE cronjob.batch/maas-api-key-cleanup */15 * * * * False 0 4m8s 29m cronjob.batch/maas-api-key-cleanup-e2e-ait-d8b57005 */15 * * * * False 0 4m8s 13m cronjob.batch/maas-api-key-cleanup-e2e-api-a-599e95 */15 * * * * False 0 4m8s 7m39s cronjob.batch/maas-api-key-cleanup-e2e-api-b-d3a40f */15 * * * * False 0 4m8s 7m31s cronjob.batch/maas-api-key-cleanup-e2e-derive-86b85e98 */15 * * * * False 0 4m8s 13m cronjob.batch/maas-api-key-cleanup-e2e-inf-a-4ec9ec */15 * * * * False 0 4m8s 9m39s cronjob.batch/maas-api-key-cleanup-e2e-inf-b-a5309f */15 * * * * False 0 4m8s 9m31s cronjob.batch/maas-api-key-cleanup-e2e-migrate-620eb2f1 */15 * * * * False 0 4m8s 13m cronjob.batch/maas-api-key-cleanup-e2e-mt-c2c3a008 */15 * * * * False 0 4m8s 11m cronjob.batch/maas-api-key-cleanup-e2e-shared-a-c664d0 */15 * * * * False 0 4m8s 6m49s cronjob.batch/maas-api-key-cleanup-e2e-shared-b-700cbf */15 * * * * False 0 4m8s 6m40s NAME STATUS COMPLETIONS DURATION AGE job.batch/maas-api-key-cleanup-29735850 Complete 1/1 9s 19m job.batch/maas-api-key-cleanup-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-ait-d8b57005-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-api-a-599e95-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-api-b-d3a40f-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-derive-86b85e98-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-inf-a-4ec9ec-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-inf-b-a5309f-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-migrate-620eb2f1-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-mt-c2c3a008-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-shared-a-c664d0-29735865 Complete 1/1 8s 4m8s job.batch/maas-api-key-cleanup-e2e-shared-b-700cbf-29735865 Complete 1/1 3s 4m8s --- RHOAI Operator namespace (redhat-ods-operator) --- --- RHOAI Applications namespace (redhat-ods-applications) --- --- DSC / DSCI --- NAME READY REASON datasciencecluster.datasciencecluster.opendatahub.io/default-dsc True NAME AGE PHASE CREATED AT dscinitialization.dscinitialization.opendatahub.io/default-dsci 33m Ready 2026-07-15T21:16:06Z --- Gateway namespace (openshift-ingress) --- NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES pod/data-science-gateway-data-science-gateway-class-6576859cf4tp4th 1/1 Running 0 32m 10.132.0.36 ip-10-0-131-113.ec2.internal pod/e2e-shared-b-700cbf-openshift-default-664dd56c5-959sh 1/1 Terminating 1 (6m25s ago) 6m43s 10.133.0.43 ip-10-0-143-128.ec2.internal pod/istiod-openshift-gateway-94bb8fbfd-v78t8 1/1 Running 0 32m 10.133.0.27 ip-10-0-143-128.ec2.internal pod/kube-auth-proxy-5cbf464cb7-8mjpx 1/1 Running 0 33m 10.132.0.30 ip-10-0-131-113.ec2.internal pod/kube-auth-proxy-5cbf464cb7-bvmzf 1/1 Running 0 33m 10.134.0.19 ip-10-0-136-220.ec2.internal pod/maas-default-gateway-openshift-default-56654dcf7-hnfjp 1/1 Running 1 (28m ago) 30m 10.132.0.51 ip-10-0-131-113.ec2.internal pod/payload-pre-processing-74584bb9f8-54x66 1/1 Running 0 29m 10.132.0.63 ip-10-0-131-113.ec2.internal pod/payload-processing-f9b8d9b9b-ftvt9 1/1 Running 0 29m 10.132.0.62 ip-10-0-131-113.ec2.internal pod/router-default-59ff8f8888-cmlk2 1/1 Running 0 39m 10.132.0.16 ip-10-0-131-113.ec2.internal NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR service/data-science-gateway-data-science-gateway-class ClusterIP 172.31.142.138 15021/TCP,443/TCP 32m gateway.networking.k8s.io/gateway-name=data-science-gateway service/istiod-openshift-gateway ClusterIP 172.31.148.202 15010/TCP,15012/TCP,443/TCP,15014/TCP 32m app=istiod,istio.io/rev=openshift-gateway service/kube-auth-proxy ClusterIP 172.31.102.117 8443/TCP,9000/TCP 33m app=kube-auth-proxy service/maas-default-gateway-openshift-default ClusterIP 172.31.115.8 15021/TCP,443/TCP 30m gateway.networking.k8s.io/gateway-name=maas-default-gateway service/payload-pre-processing ClusterIP 172.31.209.122 9004/TCP 29m app.kubernetes.io/component=api,app.kubernetes.io/name=maas-api,app.kubernetes.io/part-of=models-as-a-service,app=payload-pre-processing service/payload-processing ClusterIP 172.31.133.15 9004/TCP 29m app.kubernetes.io/component=api,app.kubernetes.io/name=maas-api,app.kubernetes.io/part-of=models-as-a-service,app=payload-processing service/router-default LoadBalancer 172.31.95.112 a31ee5ffd224740039e4fbc7ac717975-492448202.us-east-1.elb.amazonaws.com 80:31714/TCP,443:31000/TCP 46m ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default service/router-internal-default ClusterIP 172.31.169.174 80/TCP,443/TCP,1936/TCP 46m ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default --- AuthPolicies --- NAMESPACE NAME AGE openshift-ingress maas-gateway-auth 28m --- TokenRateLimitPolicies --- NAMESPACE NAME AGE llm maas-trlp-facebook-opt-125m-simulated 17m llm maas-trlp-premium-simulated-simulated-premium 27m openshift-ingress gateway-default-deny 29m openshift-ingress gateway-default-deny-e2e-ait-d8b57005 13m openshift-ingress gateway-default-deny-e2e-api-a-599e95 7m38s openshift-ingress gateway-default-deny-e2e-api-b-d3a40f 7m30s openshift-ingress gateway-default-deny-e2e-derive-86b85e98 13m openshift-ingress gateway-default-deny-e2e-inf-a-4ec9ec 9m39s openshift-ingress gateway-default-deny-e2e-inf-b-a5309f 9m31s openshift-ingress gateway-default-deny-e2e-migrate-620eb2f1 13m openshift-ingress gateway-default-deny-e2e-mt-c2c3a008 11m openshift-ingress gateway-default-deny-e2e-shared-a-c664d0 6m49s openshift-ingress gateway-default-deny-e2e-shared-b-700cbf 6m40s --- MaaS CRs (maas.opendatahub.io) --- NAME AGE default 29m NAMESPACE NAME READY TENANT NAMESPACE GATEWAY AGE ai-tenants models-as-a-service True models-as-a-service maas-default-gateway 29m NAME PHASE AGE maasauthpolicy.maas.opendatahub.io/premium-simulator-access Active 28m maasauthpolicy.maas.opendatahub.io/simulator-access Active 28m NAME PHASE PRIORITY AGE maassubscription.maas.opendatahub.io/premium-simulator-subscription Active 20 28m maassubscription.maas.opendatahub.io/simulator-subscription Active 10 17m NAME READY REASON AGE default-tenant True Reconciled 29m --- Inference CRs (inference.opendatahub.io) --- --- AI Gateway CRs --- --- HTTPRoutes --- NAMESPACE NAME HOSTNAMES AGE llm e2e-distinct-2-simulated-kserve-route 28m llm e2e-distinct-simulated-kserve-route 28m llm e2e-trlp-test-simulated-kserve-route 28m llm e2e-unconfigured-facebook-opt-125m-simulated-kserve-route 28m llm facebook-opt-125m-simulated-kserve-route 28m llm premium-simulated-simulated-premium-kserve-route 27m odh-ai-gateway-infra maas-api-route 29m odh-ai-gateway-infra maas-api-route-e2e-ait-d8b57005 13m odh-ai-gateway-infra maas-api-route-e2e-api-a-599e95 7m40s odh-ai-gateway-infra maas-api-route-e2e-api-b-d3a40f 7m32s odh-ai-gateway-infra maas-api-route-e2e-derive-86b85e98 13m odh-ai-gateway-infra maas-api-route-e2e-inf-a-4ec9ec 9m41s odh-ai-gateway-infra maas-api-route-e2e-inf-b-a5309f 9m33s odh-ai-gateway-infra maas-api-route-e2e-migrate-620eb2f1 13m odh-ai-gateway-infra maas-api-route-e2e-mt-c2c3a008 11m odh-ai-gateway-infra maas-api-route-e2e-shared-a-c664d0 6m51s odh-ai-gateway-infra maas-api-route-e2e-shared-b-700cbf 6m42s openshift-ingress oauth-callback-route 33m --- Gateway --- NAMESPACE NAME CLASS ADDRESS PROGRAMMED AGE openshift-ingress data-science-gateway data-science-gateway-class data-science-gateway-data-science-gateway-class.openshift-ingress.svc.cluster.local True 33m openshift-ingress maas-default-gateway openshift-default maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local True 30m