: map[apiVersion:datasciencecluster.opendatahub.io/v2 kind:DataScienceCluster metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{"apiVersion":"datasciencecluster.opendatahub.io/v2","kind":"DataScienceCluster","metadata":{"annotations":{},"name":"default-dsc"},"spec":{"components":{"aigateway":{"managementState":"Removed"},"aipipelines":{"managementState":"Managed"},"codeflare":{"managementState":"Managed"},"dashboard":{"managementState":"Managed"},"feastoperator":{"managementState":"Managed"},"kserve":{"defaultDeploymentMode":"RawDeployment","managementState":"Managed","serving":{"managementState":"Removed","name":"knative-serving"}},"kueue":{"managementState":"Removed"},"llamastackoperator":{"managementState":"Removed"},"mlflowoperator":{"managementState":"Managed"},"modelregistry":{"managementState":"Managed"},"ogx":{"managementState":"Removed"},"ray":{"managementState":"Managed"},"sparkoperator":{"managementState":"Removed"},"trainer":{"managementState":"Mana ged"},"trainingoperator":{"managementState":"Removed"},"trustyai":{"managementState":"Managed"},"workbenches":{"managementState":"Managed"}}}} ] creationTimestamp:2026-09-15T09:51:15Z finalizers:[platform.opendatahub.io/finalizer] generation:31 managedFields:[map[apiVersion:datasciencecluster.opendatahub.io/v2 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[f:kubectl.kubernetes.io/last-applied-configuration:map[]] f:finalizers:map[v:"platform.opendatahub.io/finalizer":map[]]] f:status:map[f:components:map[f:aigateway:map[f:managementState:map[]] f:aipipelines:map[f:managementState:map[]] f:batchGateway:map[f:managementState:map[]] f:dashboard:map[f:managementState:map[]] f:feastoperator:map[f:managementState:map[]] f:kserve:map[f:managementState:map[] f:releases:map[k:{"name":"AutoGluon"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"Caikit"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"Caikit-nlp"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"KServe"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"MLServer"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"OpenVINO Model Server"}:map[.: map[] f:name:map[] f:version:map[]] k:{"name":"Text Generation Inference Server"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"llm-d-latency-predictor"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"llm-d-router"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"llm-d-workload-variant-autoscaler"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"platform"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"vLLM"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"vLLM-Omni"}:map[.:map[] f:name:map[] f:version:map[]]]] f:kueue:map[f:managementState:map[] f:releases:map[k:{"name":"platform"}:map[.:map[] f:name:map[] f:version:map[]]]] f:llamastackoperator:map[] f:mcplifecycleoperator:map[f:managementState:map[]] f:mlflowoperator:map[f:managementState:map[] f:releases:map[k:{"name":"MLflow"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"platform"}:map[.:map[] f:name:map[] f:version:map[]]]] f:modelregistry:map[f:managementState:map[] f:registriesNamespace:map[] f :releases:map[k:{"name":"Kubeflow Model Registry"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"Open Data Hub Model Registry Operator"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"platform"}:map[.:map[] f:name:map[] f:version:map[]]]] f:modelsAsAService:map[f:managementState:map[]] f:ogx:map[f:managementState:map[]] f:ray:map[f:managementState:map[]] f:sparkoperator:map[f:managementState:map[] f:releases:map[k:{"name":"Spark Operator"}:map[.:map[] f:name:map[] f:version:map[]] k:{"name":"Spark"}:map[.:map[] f:name:map[] f:version:map[]]]] f:trainer:map[f:managementState:map[]] f:trainingoperator:map[f:managementState:map[]] f:trustyai:map[f:managementState:map[] f:releases:map[k:{"name":"EvalHub"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI Guardrails orchestrator"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI LMEval driver"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"T rustyAI LMEval job"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI builtin detectors"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI operator"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI service"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"TrustyAI sidecar gateway"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]] k:{"name":"platform"}:map[.:map[] f:name:map[] f:version:map[]]]] f:workbenches:map[f:managementState:map[] f:releases:map[k:{"name":"Kubeflow Notebook Controller"}:map[.:map[] f:name:map[] f:repoUrl:map[] f:version:map[]]] f:workbenchNamespace:map[]] f:workbenchesV2:map[f:managementState:map[]]] f:conditions:map[] f:observedGeneration:map[] f:phase:map[] f:release:map[f:name:map[] f:version:map[]]]] manager:datasciencecluster operation:Apply subresource:status time:2026-09-15T12:42:25Z] map[apiVersion:datasciencecluster.opendatahub.io/v2 fieldsType:FieldsV1 fieldsV1: map[f:metadata:map[f:annotations:map[.:map[] f:kubectl.kubernetes.io/last-applied-configuration:map[]]] f:spec:map[.:map[] f:components:map[.:map[] f:aigateway:map[] f:aipipelines:map[] f:dashboard:map[] f:feastoperator:map[] f:kserve:map[.:map[] f:modelsAsService:map[.:map[] f:managementState:map[]] f:nim:map[.:map[] f:airGapped:map[] f:managementState:map[]] f:rawDeploymentServiceConfig:map[] f:wva:map[.:map[] f:managementState:map[]]] f:kueue:map[.:map[] f:autoCreateQueues:map[] f:defaultClusterQueueName:map[] f:defaultLocalQueueName:map[]] f:llamastackoperator:map[.:map[] f:managementState:map[]] f:mlflowoperator:map[.:map[] f:managementState:map[]] f:modelregistry:map[.:map[] f:managementState:map[] f:registriesNamespace:map[]] f:ogx:map[] f:ray:map[] f:sparkoperator:map[] f:trainer:map[] f:trainingoperator:map[.:map[] f:managementState:map[]] f:trustyai:map[.:map[] f:managementState:map[] f:mcpGuardrailsMode:map[]] f:workbenches:map[.:map[] f:workbenchNamespace:map[]]]]] manager:kubectl-client-side-appl y operation:Update time:2026-09-15T09:51:15Z] map[apiVersion:datasciencecluster.opendatahub.io/v2 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:finalizers:map[.:map[] v:"platform.opendatahub.io/finalizer":map[]]] f:spec:map[f:components:map[f:aigateway:map[f:batchGateway:map[] f:modelsAsAService:map[]] f:mcplifecycleoperator:map[] f:trustyai:map[f:eval:map[.:map[] f:lmeval:map[.:map[] f:permitCodeExecution:map[] f:permitOnline:map[]]]] f:workbenches:map[f:workbenchesV2:map[.:map[] f:managementState:map[]]]]]] manager:manager operation:Update time:2026-09-15T09:51:15Z] map[apiVersion:datasciencecluster.opendatahub.io/v2 fieldsType:FieldsV1 fieldsV1:map[f:spec:map[f:components:map[f:aigateway:map[f:modelsAsAService:map[f:managementState:map[]]] f:kueue:map[f:managementState:map[]] f:sparkoperator:map[f:managementState:map[]]]]] manager:kubectl-patch operation:Update time:2026-09-15T10:13:21Z] map[apiVersion:datasciencecluster.opendatahub.io/v2 fieldsType:FieldsV1 fieldsV1:map[f:spec:map[f:components:map[f:a igateway:map[f:managementState:map[]] f:aipipelines:map[f:managementState:map[]] f:dashboard:map[f:managementState:map[]] f:feastoperator:map[f:managementState:map[]] f:kserve:map[f:managementState:map[]] f:mcplifecycleoperator:map[f:managementState:map[]] f:ogx:map[f:managementState:map[]] f:ray:map[f:managementState:map[]] f:trainer:map[f:managementState:map[]] f:workbenches:map[f:managementState:map[]]]]] manager:e2e-tests operation:Update time:2026-09-15T12:39:01Z]] name:default-dsc resourceVersion:308910401 uid:4c853b9e-b24a-4b2c-b457-36f8223550c7] spec:map[components:map[aigateway:map[batchGateway:map[] managementState:Removed modelsAsAService:map[managementState:Managed]] aipipelines:map[managementState:Removed] dashboard:map[managementState:Managed] feastoperator:map[managementState:Removed] kserve:map[managementState:Managed modelsAsService:map[managementState:Removed] nim:map[airGapped:false managementState:Managed] rawDeploymentServiceConfig:Headless wva:map[managementState:Removed]] kueue:map[auto CreateQueues:false defaultClusterQueueName:default defaultLocalQueueName:default managementState:Unmanaged] llamastackoperator:map[managementState:Removed] mcplifecycleoperator:map[managementState:Removed] mlflowoperator:map[managementState:Managed] modelregistry:map[managementState:Managed registriesNamespace:rhoai-model-registries] ogx:map[managementState:Removed] ray:map[managementState:Removed] sparkoperator:map[managementState:Managed] trainer:map[managementState:Removed] trainingoperator:map[managementState:Removed] trustyai:map[eval:map[lmeval:map[permitCodeExecution:deny permitOnline:deny]] managementState:Managed mcpGuardrailsMode:false] workbenches:map[managementState:Managed workbenchNamespace:rhods-notebooks workbenchesV2:map[managementState:Removed]]]] status:map[components:map[aigateway:map[managementState:Removed] aipipelines:map[managementState:Removed] batchGateway:map[managementState:Removed] dashboard:map[managementState:Managed] feastoperator:map[managementState:Removed] kserve:map[managem entState:Managed releases:[map[name:KServe version:v0.20.0] map[name:vLLM version:v0.26.0] map[name:llm-d-router version:v0.10.0] map[name:llm-d-workload-variant-autoscaler version:v0.9.0] map[name:llm-d-latency-predictor version:v0.9.0] map[name:vLLM-Omni version:v0.26.0] map[name:OpenVINO Model Server version:v2025.4] map[name:MLServer version:1.7.1] map[name:Caikit version:v0.27.7] map[name:Caikit-nlp version:v0.5.8] map[name:AutoGluon version:v0.1.0] map[name:Text Generation Inference Server version:commit 8a8c55dd0884b8e4e420d22f10bcce6752455edf] map[name:platform version:3.6.0-ea.2]]] kueue:map[managementState:Unmanaged releases:[map[name:platform version:3.6.0-ea.2]]] llamastackoperator:map[] mcplifecycleoperator:map[managementState:Removed] mlflowoperator:map[managementState:Managed releases:[map[name:platform version:3.6.0-ea.2] map[name:MLflow repoUrl:https://github.com/mlflow/mlflow version:3.15.2]]] modelregistry:map[managementState:Managed registriesNamespace:rhoai-model-registries releases:[map[ name:Kubeflow Model Registry repoUrl:https://github.com/kubeflow/model-registry version:v0.3.16] map[name:Open Data Hub Model Registry Operator repoUrl:https://github.com/opendatahub-io/model-registry-operator version:v0.3.16] map[name:platform version:3.6.0-ea.2]]] modelsAsAService:map[managementState:Removed] ogx:map[managementState:Removed] ray:map[managementState:Removed] sparkoperator:map[managementState:Managed releases:[map[name:Spark Operator version:v2.4.0] map[name:Spark version:4.0.1]]] trainer:map[managementState:Removed] trainingoperator:map[managementState:Removed] trustyai:map[managementState:Managed releases:[map[name:platform version:3.6.0-ea.2] map[name:TrustyAI operator repoUrl:https://github.com/trustyai-explainability/trustyai-service-operator version:v1.37.0] map[name:TrustyAI service repoUrl:https://github.com/trustyai-explainability/trustyai-explainability version:v0.28.0] map[name:TrustyAI LMEval driver repoUrl:https://github.com/trustyai-explainability/trustyai-service-operator versi on:v1.37.0] map[name:TrustyAI LMEval job repoUrl:https://github.com/EleutherAI/lm-evaluation-harness version:v0.4.8] map[name:TrustyAI Guardrails orchestrator repoUrl:https://github.com/foundation-model-stack/fms-guardrails-orchestrator version:0.9.4] map[name:TrustyAI builtin detectors repoUrl:https://github.com/trustyai-explainability/guardrails-regex-detector version:v0.2.0] map[name:TrustyAI sidecar gateway repoUrl:https://github.com/trustyai-explainability/vllm-orchestrator-gateway version:v0.2.1] map[name:EvalHub repoUrl:https://github.com/eval-hub/eval-hub version:v0.3.0]]] workbenches:map[managementState:Managed releases:[map[name:Kubeflow Notebook Controller repoUrl:https://github.com/kubeflow/kubeflow version:1.10.0]] workbenchNamespace:rhods-notebooks] workbenchesV2:map[managementState:Removed]] conditions:[map[lastTransitionTime:2026-09-15T12:42:02Z message:Some modules are not ready: dashboard, kserve, workbenches reason:NotReady status:False type:Ready] map[lastTransitionTime:2026-09-15T12:39:01 Z observedGeneration:31 status:True type:ProvisioningSucceeded] map[lastTransitionTime:2026-09-15T12:38:59Z status:True type:ComponentsReady] map[lastTransitionTime:2026-09-15T12:42:02Z message:Some modules are not ready: dashboard, kserve, workbenches reason:NotReady status:False type:ModulesReady] map[lastTransitionTime:2026-09-15T12:38:44Z message:Module ManagementState is set to Removed reason:Removed severity:Info status:False type:AIGatewayReady] map[lastTransitionTime:2026-09-15T12:42:25Z message:Component ManagementState is set to Removed reason:Removed severity:Info status:False type:AIPipelinesReady] map[lastTransitionTime:2026-09-15T09:51:18Z message:Submodule ManagementState is set to Removed reason:Removed severity:Info status:False type:BatchGatewayReady] map[lastTransitionTime:2026-09-15T12:41:45Z message:1 module(s) degraded reason:ModulesDegraded status:False type:DashboardReady] map[lastTransitionTime:2026-09-15T12:37:25Z message:Module ManagementState is set to Removed reason:Removed severi ty:Info status:False type:FeastOperatorReady] map[lastTransitionTime:2026-09-15T12:37:49Z severity:Info status:True type:KserveLLMInferenceServiceDependencies] map[lastTransitionTime:2026-09-15T12:37:49Z severity:Info status:True type:KserveLLMInferenceServiceWideEPDependencies] map[lastTransitionTime:2026-09-15T12:42:02Z message:deployments not ready: kserve-controller-manager (availableReplicas=0) reason:DeploymentNotReady status:False type:KserveReady] map[lastTransitionTime:2026-09-15T12:42:25Z status:True type:KueueReady] map[lastTransitionTime:2026-09-15T12:38:04Z message:Module ManagementState is set to Removed reason:Removed severity:Info status:False type:MCPLifecycleOperatorReady] map[lastTransitionTime:2026-09-15T09:51:47Z message:MLflowOperator is ready to manage MLflow custom resources reason:Ready status:True type:MLflowOperatorReady] map[lastTransitionTime:2026-09-15T09:52:12Z reason:AllDependentsHealthy status:True type:ModelRegistryReady] map[lastTransitionTime:2026-09-15T12:38:44Z message:Su bmodule ManagementState is set to Removed reason:Removed severity:Info status:False type:ModelsAsAServiceReady] map[lastTransitionTime:2026-09-15T12:38:36Z message:Module ManagementState is set to Removed reason:Removed severity:Info status:False type:OGXReady] map[lastTransitionTime:2026-09-15T12:42:25Z message:Component ManagementState is set to Removed reason:Removed severity:Info status:False type:RayReady] map[lastTransitionTime:2026-09-15T10:13:16Z reason:AllDependentsHealthy status:True type:SparkOperatorReady] map[lastTransitionTime:2026-09-15T12:38:35Z message:Module ManagementState is set to Removed reason:Removed severity:Info status:False type:TrainerReady] map[lastTransitionTime:2026-09-15T12:42:25Z status:True type:TrustyAIReady] map[lastTransitionTime:2026-09-15T12:42:25Z message:deployment redhat-ods-applications/notebook-controller-deployment has 0/1 ready replicas. Warning: 18 ImageStream tag(s) failed to import: s2i-minimal-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rh oai/odh-workbench-jupyter-minimal-cpu-py312-rhe...); tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-cuda-py312...); jupyter-rocm-tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-rocm-py312...); minimal-gpu:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-cuda-py312-rh...); runtime-pytorch:pytorch (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-cuda-py312-rhe...); runtime-pytorch-llmcompressor:pytorch-llmcompressor (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-llmcompressor-...); code-server-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-codeserver-datascience-cpu-py...); jupyter-rocm-pytorch:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-rocm-py312-rh...); odh-trustyai-notebook:3.5 (dockerimag e.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-trustyai-cpu-py312-rh...); pytorch:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-cuda-py312-rh...); ... and 8 more reason:DeploymentsNotReady status:False type:WorkbenchesReady] map[lastTransitionTime:2026-09-15T09:51:18Z message:Submodule ManagementState is set to Removed reason:Removed severity:Info status:False type:WorkbenchesV2Ready]] observedGeneration:7 phase:Not Ready release:map[name:OpenShift AI Self-Managed version:3.6.0-ea.2]]] to match expression : .status.conditions[] | select(.type == "WorkbenchesReady") | .status == "True" testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]> : DataScienceCluster.datasciencecluster.opendatahub.io "default-dsc" is invalid: spec.components.modelregistry: Invalid value: "object": RegistriesNamespace is immutable when model registry is Managed { ErrStatus: { TypeMeta: {Kind: "Status", APIVersion: "v1"}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, ShardInfo: nil, }, Status: "Failure", Message: "DataScienceCluster.datasciencecluster.opendatahub.io \"default-dsc\" is invalid: spec.components.modelregistry: Invalid value: \"object\": RegistriesNamespace is immutable when model registry is Managed", Reason: "Invalid", Details: { Name: "default-dsc", Group: "datasciencecluster.opendatahub.io", Kind: "DataScienceCluster", UID: "", Causes: [ { Type: "FieldValueInvalid", Message: "Invalid value: \"object\": RegistriesNamespace is immutable when model registry is Managed", Field: "spec.components.modelregistry", }, ], RetryAfterSeconds: 0, }, Code: 422, }, } occurred testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test 2026/09/15 12:52:51 TEST FAILURE DETECTED: group 2 2026/09/15 12:52:51 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 2) === 2026/09/15 12:52:51 === CLUSTER STATE === 2026/09/15 12:52:51 Node: ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 12:52:51 Condition MemoryPressure: False - kubelet has sufficient memory available 2026/09/15 12:52:51 Condition DiskPressure: False - kubelet has no disk pressure 2026/09/15 12:52:51 Condition PIDPressure: False - kubelet has sufficient PID available 2026/09/15 12:52:51 Condition Ready: True - kubelet is posting ready status 2026/09/15 12:52:51 Resources: 7500m CPU, 31236232Ki memory allocatable (of 8000m CPU, 32387208Ki memory capacity) 2026/09/15 12:52:51 Node: ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 12:52:51 Condition MemoryPressure: False - kubelet has sufficient memory available 2026/09/15 12:52:51 Condition DiskPressure: False - kubelet has no disk pressure 2026/09/15 12:52:51 Condition PIDPressure: False - kubelet has sufficient PID available 2026/09/15 12:52:51 Condition Ready: True - kubelet is posting ready status 2026/09/15 12:52:51 Resources: 7500m CPU, 31236232Ki memory allocatable (of 8000m CPU, 32387208Ki memory capacity) 2026/09/15 12:52:51 === DEPLOYMENTS === 2026/09/15 12:52:51 Namespace: redhat-ods-applications 2026/09/15 12:52:51 Deployment aihub-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment catalog-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment mlflow-operator-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment model-registry-operator-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment notebook-controller-deployment: 1/1 (READY) 2026/09/15 12:52:51 Deployment odh-notebook-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment odh-observability: 1/1 (READY) 2026/09/15 12:52:51 Deployment spark-operator-controller: 1/1 (READY) 2026/09/15 12:52:51 Deployment spark-operator-module-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment spark-operator-webhook: 1/1 (READY) 2026/09/15 12:52:51 Deployment trustyai-service-operator-controller-manager: 1/1 (READY) 2026/09/15 12:52:51 Deployment workbenches-operator: 1/1 (READY) 2026/09/15 12:52:51 Namespace: kube-system 2026/09/15 12:52:51 === PODS === 2026/09/15 12:52:51 Failed to collect pod data: redhat-ods-applications/aihub-controller-manager-77fc779c49-76nmn: container manager restarts=1; redhat-ods-applications/model-registry-operator-controller-manager-6958f49b4-q2wnv: phase=Pending; redhat-ods-applications/workbenches-operator-6d6754bf4-qr4jp: container manager restarts=1 2026/09/15 12:52:51 Continuing with partial pod data to collect logs from problematic containers 2026/09/15 12:52:51 Namespace kube-system: no problematic pods found 2026/09/15 12:52:51 Namespace: redhat-ods-applications 2026/09/15 12:52:51 Pod aihub-controller-manager-77fc779c49-76nmn: Running 2026/09/15 12:52:51 Container manager: ready=true, restarts=1 2026/09/15 12:52:51 Pod model-registry-operator-controller-manager-6958f49b4-q2wnv: Pending 2026/09/15 12:52:51 Container manager: ready=false, waiting: ImagePullBackOff Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown 2026/09/15 12:52:51 Pod workbenches-operator-6d6754bf4-qr4jp: Running 2026/09/15 12:52:51 Container manager: ready=true, restarts=1 2026/09/15 12:52:51 === LOGS (CURRENT) for container manager === 2026/09/15 12:52:51 Failed to retrieve logs: error opening log stream: container "manager" in pod "model-registry-operator-controller-manager-6958f49b4-q2wnv" is waiting to start: trying and failing to pull image 2026/09/15 12:52:51 === OPERATOR STATUS === 2026/09/15 12:52:51 Operator deployment: 3/3 ready 2026/09/15 12:52:51 === DSCI STATUS === 2026/09/15 12:52:51 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False 2026/09/15 12:52:51 === DSC STATUS === 2026/09/15 12:52:51 DSC default-dsc: 2026/09/15 12:52:51 Ready: True - 2026/09/15 12:52:51 ProvisioningSucceeded: True - 2026/09/15 12:52:51 ComponentsReady: True - 2026/09/15 12:52:51 ModulesReady: True - 2026/09/15 12:52:51 AIGatewayReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 AIPipelinesReady: False - Component ManagementState is set to Removed 2026/09/15 12:52:51 BatchGatewayReady: False - Submodule ManagementState is set to Removed 2026/09/15 12:52:51 DashboardReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 FeastOperatorReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 KserveLLMInferenceServiceDependencies: False - Submodule ManagementState is set to Removed 2026/09/15 12:52:51 KserveLLMInferenceServiceWideEPDependencies: False - Submodule ManagementState is set to Removed 2026/09/15 12:52:51 KserveReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 KueueReady: False - Component ManagementState is set to Removed 2026/09/15 12:52:51 MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 MLflowOperatorReady: True - MLflowOperator is ready to manage MLflow custom resources 2026/09/15 12:52:51 ModelRegistryReady: True - 2026/09/15 12:52:51 ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed 2026/09/15 12:52:51 OGXReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 RayReady: False - Component ManagementState is set to Removed 2026/09/15 12:52:51 SparkOperatorReady: True - 2026/09/15 12:52:51 TrainerReady: False - Module ManagementState is set to Removed 2026/09/15 12:52:51 TrustyAIReady: True - 2026/09/15 12:52:51 WorkbenchesReady: True - Workbenches component is ready. Warning: 18 ImageStream tag(s) failed to import: runtime-rocm-pytorch:rocm-pytorch (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-rocm-py312-rhe...); runtime-tensorflow:tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-cuda-py312-...); s2i-generic-data-science-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-datascience-cpu-py312...); runtime-minimal:minimal (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-minimal-cpu-py312-rhel...); jupyter-pytorch-llmcompressor:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-llmcompressor...); jupyter-rocm-minimal:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-rocm-py312-rh...); runtime-datascience:datascience (dockerimage.image.openshift.io "regi stry.redhat.io/rhoai/odh-pipeline-runtime-datascience-cpu-py312-...); s2i-minimal-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-cpu-py312-rhe...); tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-cuda-py312...); jupyter-rocm-tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-rocm-py312...); ... and 8 more 2026/09/15 12:52:51 WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed 2026/09/15 12:52:51 === RECENT EVENTS === 2026/09/15 12:52:51 12:50:31 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: Started - Started container manager 2026/09/15 12:52:51 12:50:31 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:21d0e874e64559529e54ecb8ba105093ef698a3a60846c22bfb691c10245d92c" in 1.537s (1.537s including waiting). Image size: 168442549 bytes. 2026/09/15 12:52:51 12:50:31 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: Created - Created container: manager 2026/09/15 12:52:51 12:50:29 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-b45c57cdb-kg4sb to ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 12:52:51 12:50:29 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:21d0e874e64559529e54ecb8ba105093ef698a3a60846c22bfb691c10245d92c" 2026/09/15 12:52:51 12:50:29 INFO Pod/notebook-controller-deployment-b45c57cdb-kg4sb: AddedInterface - Add eth0 [10.130.21.46/23] from ovn-kubernetes 2026/09/15 12:52:51 12:50:27 WARN Pod/notebook-controller-deployment-b45c57cdb-kg4sb: FailedScheduling - 0/3 nodes are available: 1 node(s) were unschedulable, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod. 2026/09/15 12:52:51 12:50:26 WARN Pod/notebook-controller-deployment-b45c57cdb-kg4sb: FailedScheduling - 0/3 nodes are available: 1 node(s) were unschedulable, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod. 2026/09/15 12:52:51 12:50:26 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Preempted - Preempted by pod a2fe2b99-19d4-4834-b6fd-515addeead87 on node ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 12:52:51 12:50:26 INFO ReplicaSet/notebook-controller-deployment-b45c57cdb: SuccessfulCreate - Created pod: notebook-controller-deployment-b45c57cdb-kg4sb 2026/09/15 12:52:51 12:50:26 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Killing - Stopping container manager 2026/09/15 12:52:51 12:50:25 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 12:52:51 12:50:25 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 12:52:51 12:48:18 WARN Pod/model-registry-operator-controller-manager-6958f49b4-q2wnv: Failed - Error: ImagePullBackOff 2026/09/15 12:52:51 12:48:18 INFO Pod/model-registry-operator-controller-manager-6958f49b4-q2wnv: BackOff - Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646" 2026/09/15 12:52:51 12:48:12 WARN Pod/rhods-dashboard-55c678f9b-58ssh: Unhealthy - Readiness probe failed: Get "https://10.130.21.40:8444/healthz": dial tcp 10.130.21.40:8444: connect: connection refused 2026/09/15 12:52:51 12:48:12 WARN Pod/rhods-dashboard-6bcf96c49-9rhjb: Unhealthy - Readiness probe failed: Get "https://10.130.21.38:8943/healthcheck": dial tcp 10.130.21.38:8943: connect: connection refused 2026/09/15 12:52:51 12:48:12 WARN Pod/rhods-dashboard-6bcf96c49-9rhjb: Unhealthy - Readiness probe failed: Get "https://10.130.21.38:8444/healthz": dial tcp 10.130.21.38:8444: connect: connection refused 2026/09/15 12:52:51 12:48:12 WARN Pod/rhods-dashboard-55c678f9b-58ssh: Unhealthy - Readiness probe failed: Get "https://10.130.21.40:8943/healthcheck": dial tcp 10.130.21.40:8943: connect: connection refused 2026/09/15 12:52:51 12:48:10 WARN Pod/rhods-dashboard-6bcf96c49-9rhjb: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1 2026/09/15 12:52:51 12:48:10 WARN Pod/rhods-dashboard-55c678f9b-58ssh: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1 2026/09/15 12:52:51 12:48:05 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Created - Created container: manager 2026/09/15 12:52:51 12:48:05 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:21d0e874e64559529e54ecb8ba105093ef698a3a60846c22bfb691c10245d92c" in 1.145s (1.145s including waiting). Image size: 168442549 bytes. 2026/09/15 12:52:51 12:48:05 WARN Pod/rhods-dashboard-6bcf96c49-dsxrz: Unhealthy - Readiness probe failed: Get "https://10.130.21.39:8444/healthz": dial tcp 10.130.21.39:8444: connect: connection refused 2026/09/15 12:52:51 12:48:05 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Started - Started container manager 2026/09/15 12:52:51 12:48:05 WARN Pod/rhods-dashboard-6bcf96c49-dsxrz: Unhealthy - Readiness probe failed: Get "https://10.130.21.39:8943/healthcheck": dial tcp 10.130.21.39:8943: connect: connection refused 2026/09/15 12:52:51 12:48:04 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: AddedInterface - Add eth0 [10.130.21.41/23] from ovn-kubernetes 2026/09/15 12:52:51 12:48:04 WARN Pod/rhods-dashboard-6bcf96c49-dsxrz: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1 2026/09/15 12:52:51 12:48:04 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:21d0e874e64559529e54ecb8ba105093ef698a3a60846c22bfb691c10245d92c" 2026/09/15 12:52:51 12:48:03 INFO Pod/notebook-controller-deployment-b45c57cdb-9p58g: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-b45c57cdb-9p58g to ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 12:52:51 12:48:01 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 12:52:51 12:48:01 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 12:52:51 12:48:01 WARN ServiceMonitor/trustyai-service-operator-service-monitor: InvalidConfiguration - "redhat-ods-applications/trustyai-service-operator-service-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 12:52:51 === RESOURCE QUOTAS === 2026/09/15 12:52:51 No resource quotas found in monitored namespaces 2026/09/15 12:52:51 Classification for group 2: infrastructure/image-pull (code=1001, confidence=medium) 2026/09/15 12:52:51 Evidence: container model-registry-operator-controller-manager-6958f49b4-q2wnv/manager waiting: ImagePullBackOff Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"image-pull","error_code":1001,"evidence":["container model-registry-operator-controller-manager-6958f49b4-q2wnv/manager waiting: ImagePullBackOff Back-off pulling image \"registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown"],"confidence":"medium"} 2026/09/15 12:52:51 Diagnostics complete for failed test: group 2 controller_test.go:769: Stopping: group 2 test failed. 2026/09/15 12:52:51 Circuit breaker: failure classified as infrastructure/image-pull (code=1001) 2026/09/15 12:52:51 Circuit breaker: infrastructure/unknown failure recorded (2/3) 2026-09-15T12:58:57Z INFO Cluster config {"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.2"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}} 2026/09/15 13:03:00 TEST FAILURE DETECTED: components 2026/09/15 13:03:00 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) === 2026/09/15 13:03:00 === CLUSTER STATE === 2026/09/15 13:03:00 Failed to collect node data: unhealthy nodes: ip-10-0-1-158.us-east-2.compute.internal (Ready=False) 2026/09/15 13:03:00 === DEPLOYMENTS === 2026/09/15 13:03:00 Namespace: redhat-ods-applications 2026/09/15 13:03:00 Deployment aihub-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment catalog-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment kserve-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment kserve-module-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment kubeflow-trainer-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment llmisvc-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment model-registry-operator-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment model-serving-api: 1/1 (READY) 2026/09/15 13:03:00 Deployment notebook-controller-deployment: 1/1 (READY) 2026/09/15 13:03:00 Deployment odh-model-controller: 1/1 (READY) 2026/09/15 13:03:00 Deployment odh-notebook-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment odh-observability: 1/1 (READY) 2026/09/15 13:03:00 Deployment spark-operator-controller: 1/1 (READY) 2026/09/15 13:03:00 Deployment spark-operator-module-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment spark-operator-webhook: 1/1 (READY) 2026/09/15 13:03:00 Deployment trainer-operator-controller-manager: 1/1 (READY) 2026/09/15 13:03:00 Deployment workbenches-operator: 1/1 (READY) 2026/09/15 13:03:00 Namespace: kube-system 2026/09/15 13:03:00 === PODS === 2026/09/15 13:03:00 Failed to collect pod data: redhat-ods-applications/aihub-controller-manager-77fc779c49-76nmn: container manager restarts=1; redhat-ods-applications/model-registry-operator-controller-manager-6958f49b4-q2wnv: phase=Pending; redhat-ods-applications/workbenches-operator-6d6754bf4-qr4jp: container manager restarts=1; kube-system/global-pull-secret-syncer-ch6px: phase=Pending 2026/09/15 13:03:00 Continuing with partial pod data to collect logs from problematic containers 2026/09/15 13:03:00 Namespace: kube-system 2026/09/15 13:03:00 Pod global-pull-secret-syncer-ch6px: Pending 2026/09/15 13:03:00 Container global-pull-secret-syncer: ready=false, waiting: ContainerCreating 2026/09/15 13:03:00 Namespace: redhat-ods-applications 2026/09/15 13:03:00 Pod aihub-controller-manager-77fc779c49-76nmn: Running 2026/09/15 13:03:00 Container manager: ready=true, restarts=1 2026/09/15 13:03:00 Pod model-registry-operator-controller-manager-6958f49b4-q2wnv: Pending 2026/09/15 13:03:00 Container manager: ready=false, waiting: ImagePullBackOff Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown 2026/09/15 13:03:00 Pod workbenches-operator-6d6754bf4-qr4jp: Running 2026/09/15 13:03:00 Container manager: ready=true, restarts=1 2026/09/15 13:03:00 === LOGS (CURRENT) for container manager === 2026/09/15 13:03:00 Failed to retrieve logs: error opening log stream: container "manager" in pod "model-registry-operator-controller-manager-6958f49b4-q2wnv" is waiting to start: trying and failing to pull image 2026/09/15 13:03:00 === OPERATOR STATUS === 2026/09/15 13:03:00 Operator deployment: 3/3 ready 2026/09/15 13:03:00 === DSCI STATUS === 2026/09/15 13:03:00 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False 2026/09/15 13:03:00 === DSC STATUS === 2026/09/15 13:03:00 DSC default-dsc: 2026/09/15 13:03:00 Ready: True - 2026/09/15 13:03:00 ProvisioningSucceeded: True - 2026/09/15 13:03:00 ComponentsReady: True - All registered components have ManagementState Removed or are not configured 2026/09/15 13:03:00 ModulesReady: True - 2026/09/15 13:03:00 AIGatewayReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 AIPipelinesReady: False - Component ManagementState is set to Removed 2026/09/15 13:03:00 BatchGatewayReady: False - Submodule ManagementState is set to Removed 2026/09/15 13:03:00 DashboardReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 FeastOperatorReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 KserveLLMInferenceServiceDependencies: True - 2026/09/15 13:03:00 KserveLLMInferenceServiceWideEPDependencies: True - 2026/09/15 13:03:00 KserveReady: True - 2026/09/15 13:03:00 KueueReady: False - Component ManagementState is set to Removed 2026/09/15 13:03:00 MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 MLflowOperatorReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 ModelRegistryReady: True - 2026/09/15 13:03:00 ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed 2026/09/15 13:03:00 OGXReady: False - Module ManagementState is set to Removed 2026/09/15 13:03:00 RayReady: False - Component ManagementState is set to Removed 2026/09/15 13:03:00 SparkOperatorReady: True - 2026/09/15 13:03:00 TrainerReady: True - 2026/09/15 13:03:00 TrustyAIReady: False - Component ManagementState is set to Removed 2026/09/15 13:03:00 WorkbenchesReady: True - Workbenches component is ready. Warning: 18 ImageStream tag(s) failed to import: runtime-rocm-tensorflow:rocm-tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-rocm-py312-...); runtime-rocm-pytorch:rocm-pytorch (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-rocm-py312-rhe...); runtime-tensorflow:tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-cuda-py312-...); s2i-generic-data-science-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-datascience-cpu-py312...); runtime-minimal:minimal (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-minimal-cpu-py312-rhel...); jupyter-pytorch-llmcompressor:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-llmcompressor...); jupyter-rocm-minimal:3.5 (dockerimage.image.openshift. io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-rocm-py312-rh...); runtime-datascience:datascience (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-datascience-cpu-py312-...); s2i-minimal-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-cpu-py312-rhe...); tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-cuda-py312...); ... and 8 more 2026/09/15 13:03:00 WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed 2026/09/15 13:03:00 === RECENT EVENTS === 2026/09/15 13:03:00 13:02:15 WARN Pod/global-pull-secret-syncer-ch6px: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started? 2026/09/15 13:03:00 13:02:08 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6dffbf4d96-c7zsd_8f00ef81-5f25-440f-9697-e8ebf8bf7a84 became leader 2026/09/15 13:03:00 13:02:06 WARN Pod/global-pull-secret-syncer-ch6px: FailedMount - MountVolume.SetUp failed for volume "original-pull-secret" : object "kube-system"/"original-pull-secret" not registered 2026/09/15 13:03:00 13:02:06 WARN Pod/global-pull-secret-syncer-ch6px: FailedMount - MountVolume.SetUp failed for volume "global-pull-secret" : object "kube-system"/"global-pull-secret" not registered 2026/09/15 13:03:00 13:02:03 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-8754f79ff-tqvp5_5ef1a975-cbb8-4548-aaaa-9fefbd995182 became leader 2026/09/15 13:03:00 13:02:03 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-78f748ccdf-hqt64_d023f0fb-04cf-4996-a402-7eff7d731ce8 became leader 2026/09/15 13:03:00 13:02:01 INFO Pod/konnectivity-agent-d8v5t: Started - Started container konnectivity-agent 2026/09/15 13:03:00 13:02:01 INFO Pod/konnectivity-agent-d8v5t: Created - Created container: konnectivity-agent 2026/09/15 13:03:00 13:02:00 INFO Pod/kube-apiserver-proxy-ip-10-0-1-158.us-east-2.compute.internal: Started - Started container haproxy 2026/09/15 13:03:00 13:02:00 INFO Pod/kube-apiserver-proxy-ip-10-0-1-158.us-east-2.compute.internal: Created - Created container: haproxy 2026/09/15 13:03:00 13:02:00 INFO Pod/kube-apiserver-proxy-ip-10-0-1-158.us-east-2.compute.internal: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4" in 6.999s (6.999s including waiting). Image size: 482987592 bytes. 2026/09/15 13:03:00 13:02:00 INFO Pod/konnectivity-agent-d8v5t: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b" in 4.905s (4.905s including waiting). Image size: 468939173 bytes. 2026/09/15 13:03:00 13:01:55 INFO Pod/konnectivity-agent-d8v5t: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b" 2026/09/15 13:03:00 13:01:53 INFO Pod/kube-apiserver-proxy-ip-10-0-1-158.us-east-2.compute.internal: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4" 2026/09/15 13:03:00 13:01:51 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: Started - Started container manager 2026/09/15 13:03:00 13:01:51 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:9e7ffbc0d6b474aa759feb6409de58d841a32e25e5e2e86d7b923bc754d84272" in 5.669s (5.669s including waiting). Image size: 235369934 bytes. 2026/09/15 13:03:00 13:01:51 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: Created - Created container: manager 2026/09/15 13:03:00 13:01:50 INFO Pod/global-pull-secret-syncer-ch6px: Scheduled - Successfully assigned kube-system/global-pull-secret-syncer-ch6px to ip-10-0-1-158.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:50 INFO Pod/konnectivity-agent-d8v5t: Scheduled - Successfully assigned kube-system/konnectivity-agent-d8v5t to ip-10-0-1-158.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:50 INFO DaemonSet/global-pull-secret-syncer: SuccessfulCreate - Created pod: global-pull-secret-syncer-ch6px 2026/09/15 13:03:00 13:01:50 INFO DaemonSet/konnectivity-agent: SuccessfulCreate - Created pod: konnectivity-agent-d8v5t 2026/09/15 13:03:00 13:01:46 INFO Pod/model-serving-api-74db48b547-rbf2q: Started - Started container server 2026/09/15 13:03:00 13:01:46 INFO Pod/odh-model-controller-8754f79ff-tqvp5: Started - Started container manager 2026/09/15 13:03:00 13:01:46 INFO Pod/odh-model-controller-8754f79ff-tqvp5: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:a92306bca53a39972e87df505dd66c7165a1e2bbd39b0ca49881db790c883006" in 703ms (703ms including waiting). Image size: 235603578 bytes. 2026/09/15 13:03:00 13:01:46 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:c90681bafc90b753b72f71ce873899fd304a44af65f92605e3283de7bae7282c" in 866ms (866ms including waiting). Image size: 247701913 bytes. 2026/09/15 13:03:00 13:01:46 INFO Pod/model-serving-api-74db48b547-rbf2q: AddedInterface - Add eth0 [10.129.71.99/23] from ovn-kubernetes 2026/09/15 13:03:00 13:01:46 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: Created - Created container: manager 2026/09/15 13:03:00 13:01:46 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: Started - Started container manager 2026/09/15 13:03:00 13:01:46 INFO Pod/model-serving-api-74db48b547-rbf2q: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:ad1c5ec4f15df0dff60ef6d93886e674b4bc512d20da31598b84ac5a009a13b3" already present on machine 2026/09/15 13:03:00 13:01:46 INFO Pod/model-serving-api-74db48b547-rbf2q: Created - Created container: server 2026/09/15 13:03:00 13:01:46 INFO Pod/odh-model-controller-8754f79ff-tqvp5: Created - Created container: manager 2026/09/15 13:03:00 13:01:45 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: AddedInterface - Add eth0 [10.129.71.98/23] from ovn-kubernetes 2026/09/15 13:03:00 13:01:45 INFO Pod/odh-model-controller-8754f79ff-tqvp5: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:a92306bca53a39972e87df505dd66c7165a1e2bbd39b0ca49881db790c883006" 2026/09/15 13:03:00 13:01:45 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:9e7ffbc0d6b474aa759feb6409de58d841a32e25e5e2e86d7b923bc754d84272" 2026/09/15 13:03:00 13:01:45 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:c90681bafc90b753b72f71ce873899fd304a44af65f92605e3283de7bae7282c" 2026/09/15 13:03:00 13:01:45 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: AddedInterface - Add eth0 [10.129.71.97/23] from ovn-kubernetes 2026/09/15 13:03:00 13:01:45 INFO Pod/odh-model-controller-8754f79ff-tqvp5: AddedInterface - Add eth0 [10.130.21.55/23] from ovn-kubernetes 2026/09/15 13:03:00 13:01:44 INFO Pod/odh-model-controller-8754f79ff-tqvp5: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-8754f79ff-tqvp5 to ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:44 INFO Pod/model-serving-api-74db48b547-rbf2q: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-74db48b547-rbf2q to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:44 INFO Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6dffbf4d96-c7zsd to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:44 INFO Pod/kserve-controller-manager-78f748ccdf-hqt64: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-78f748ccdf-hqt64 to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 13:01:44 INFO ReplicaSet/kserve-controller-manager-78f748ccdf: SuccessfulCreate - Created pod: kserve-controller-manager-78f748ccdf-hqt64 2026/09/15 13:03:00 13:01:44 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-74db48b547 from 0 to 1 2026/09/15 13:03:00 13:01:44 WARN Pod/model-serving-api-74db48b547-rbf2q: FailedMount - MountVolume.SetUp failed for volume "tls-certs" : secret "model-serving-api-tls" not found 2026/09/15 13:03:00 13:01:44 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-8754f79ff from 0 to 1 2026/09/15 13:03:00 13:01:44 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 13:01:44 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6dffbf4d96 from 0 to 1 2026/09/15 13:03:00 13:01:44 INFO ReplicaSet/llmisvc-controller-manager-6dffbf4d96: SuccessfulCreate - Created pod: llmisvc-controller-manager-6dffbf4d96-c7zsd 2026/09/15 13:03:00 13:01:44 INFO ReplicaSet/model-serving-api-74db48b547: SuccessfulCreate - Created pod: model-serving-api-74db48b547-rbf2q 2026/09/15 13:03:00 13:01:44 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-78f748ccdf from 0 to 1 2026/09/15 13:03:00 13:01:44 INFO ReplicaSet/odh-model-controller-8754f79ff: SuccessfulCreate - Created pod: odh-model-controller-8754f79ff-tqvp5 2026/09/15 13:03:00 13:01:44 WARN Pod/llmisvc-controller-manager-6dffbf4d96-c7zsd: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "llmisvc-webhook-server-cert" not found 2026/09/15 13:03:00 13:01:15 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-858fdb6fd-2v8cx_25ae07ef-6ea8-4ced-b0da-dcf9ed0903cd became leader 2026/09/15 13:03:00 13:00:50 INFO Pod/kserve-module-controller-manager-858fdb6fd-2v8cx: Created - Created container: manager 2026/09/15 13:03:00 13:00:50 INFO Pod/kserve-module-controller-manager-858fdb6fd-2v8cx: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:7caa7a001cc41b2cdb15f529cde6e0a3b3bc8784e959fb935716f9d312485abc" already present on machine 2026/09/15 13:03:00 13:00:50 INFO Pod/kserve-module-controller-manager-858fdb6fd-2v8cx: Started - Started container manager 2026/09/15 13:03:00 13:00:50 INFO Pod/kserve-module-controller-manager-858fdb6fd-2v8cx: AddedInterface - Add eth0 [10.129.71.96/23] from ovn-kubernetes 2026/09/15 13:03:00 13:00:49 INFO Pod/kserve-module-controller-manager-858fdb6fd-2v8cx: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-858fdb6fd-2v8cx to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 13:00:49 INFO ReplicaSet/kserve-module-controller-manager-858fdb6fd: SuccessfulCreate - Created pod: kserve-module-controller-manager-858fdb6fd-2v8cx 2026/09/15 13:03:00 13:00:49 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-858fdb6fd from 0 to 1 2026/09/15 13:03:00 13:00:48 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Killing - Stopping container manager 2026/09/15 13:03:00 13:00:48 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Killing - Stopping container manager 2026/09/15 13:03:00 13:00:48 INFO Pod/odh-model-controller-8754f79ff-2kh26: Killing - Stopping container manager 2026/09/15 13:03:00 13:00:48 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: Killing - Stopping container manager 2026/09/15 13:03:00 13:00:48 INFO Pod/model-serving-api-74db48b547-sqlr5: Killing - Stopping container server 2026/09/15 13:03:00 13:00:47 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 13:00:18 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6dffbf4d96-726h5_0f0cb75d-2586-4280-9cd8-682a7c341220 became leader 2026/09/15 13:03:00 13:00:17 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-8754f79ff-2kh26_6f476596-34c0-4bd2-9d02-a29144383dbb became leader 2026/09/15 13:03:00 13:00:14 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-78f748ccdf-nsvhg_66d0f766-ff32-4a0f-9be0-301aeb686c3a became leader 2026/09/15 13:03:00 13:00:01 INFO Pod/odh-model-controller-8754f79ff-2kh26: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:a92306bca53a39972e87df505dd66c7165a1e2bbd39b0ca49881db790c883006" in 4.41s (4.41s including waiting). Image size: 235603578 bytes. 2026/09/15 13:03:00 13:00:01 INFO Pod/odh-model-controller-8754f79ff-2kh26: Started - Started container manager 2026/09/15 13:03:00 13:00:01 INFO Pod/odh-model-controller-8754f79ff-2kh26: Created - Created container: manager 2026/09/15 13:03:00 12:59:58 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:9e7ffbc0d6b474aa759feb6409de58d841a32e25e5e2e86d7b923bc754d84272" in 1.062s (1.062s including waiting). Image size: 235369934 bytes. 2026/09/15 13:03:00 12:59:58 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Started - Started container manager 2026/09/15 13:03:00 12:59:58 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Started - Started container manager 2026/09/15 13:03:00 12:59:58 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Created - Created container: manager 2026/09/15 13:03:00 12:59:58 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:c90681bafc90b753b72f71ce873899fd304a44af65f92605e3283de7bae7282c" in 811ms (811ms including waiting). Image size: 247701913 bytes. 2026/09/15 13:03:00 12:59:58 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Created - Created container: manager 2026/09/15 13:03:00 12:59:57 INFO Pod/odh-model-controller-8754f79ff-2kh26: AddedInterface - Add eth0 [10.129.71.95/23] from ovn-kubernetes 2026/09/15 13:03:00 12:59:57 INFO Pod/model-serving-api-74db48b547-sqlr5: AddedInterface - Add eth0 [10.129.71.94/23] from ovn-kubernetes 2026/09/15 13:03:00 12:59:57 INFO Pod/model-serving-api-74db48b547-sqlr5: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:ad1c5ec4f15df0dff60ef6d93886e674b4bc512d20da31598b84ac5a009a13b3" already present on machine 2026/09/15 13:03:00 12:59:57 INFO Pod/model-serving-api-74db48b547-sqlr5: Created - Created container: server 2026/09/15 13:03:00 12:59:57 INFO Pod/model-serving-api-74db48b547-sqlr5: Started - Started container server 2026/09/15 13:03:00 12:59:57 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: AddedInterface - Add eth0 [10.130.21.52/23] from ovn-kubernetes 2026/09/15 13:03:00 12:59:57 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: AddedInterface - Add eth0 [10.129.71.93/23] from ovn-kubernetes 2026/09/15 13:03:00 12:59:57 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:c90681bafc90b753b72f71ce873899fd304a44af65f92605e3283de7bae7282c" 2026/09/15 13:03:00 12:59:57 INFO Pod/odh-model-controller-8754f79ff-2kh26: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:a92306bca53a39972e87df505dd66c7165a1e2bbd39b0ca49881db790c883006" 2026/09/15 13:03:00 12:59:57 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:9e7ffbc0d6b474aa759feb6409de58d841a32e25e5e2e86d7b923bc754d84272" 2026/09/15 13:03:00 12:59:56 INFO Pod/odh-model-controller-8754f79ff-2kh26: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-8754f79ff-2kh26 to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:59:56 INFO Pod/model-serving-api-74db48b547-sqlr5: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-74db48b547-sqlr5 to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:59:56 INFO Pod/llmisvc-controller-manager-6dffbf4d96-726h5: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6dffbf4d96-726h5 to ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 13:03:00 12:59:56 INFO Pod/kserve-controller-manager-78f748ccdf-nsvhg: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-78f748ccdf-nsvhg to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:59:56 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-8754f79ff from 0 to 1 2026/09/15 13:03:00 12:59:56 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6dffbf4d96 from 0 to 1 2026/09/15 13:03:00 12:59:56 INFO ReplicaSet/kserve-controller-manager-78f748ccdf: SuccessfulCreate - Created pod: kserve-controller-manager-78f748ccdf-nsvhg 2026/09/15 13:03:00 12:59:56 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 12:59:56 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-78f748ccdf from 0 to 1 2026/09/15 13:03:00 12:59:56 INFO ReplicaSet/model-serving-api-74db48b547: SuccessfulCreate - Created pod: model-serving-api-74db48b547-sqlr5 2026/09/15 13:03:00 12:59:56 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-74db48b547 from 0 to 1 2026/09/15 13:03:00 12:59:56 WARN Pod/kserve-controller-manager-78f748ccdf-nsvhg: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kserve-webhook-server-cert" not found 2026/09/15 13:03:00 12:59:56 INFO ReplicaSet/odh-model-controller-8754f79ff: SuccessfulCreate - Created pod: odh-model-controller-8754f79ff-2kh26 2026/09/15 13:03:00 12:59:56 INFO ReplicaSet/llmisvc-controller-manager-6dffbf4d96: SuccessfulCreate - Created pod: llmisvc-controller-manager-6dffbf4d96-726h5 2026/09/15 13:03:00 12:59:24 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-858fdb6fd-zqr4b_f7972bb6-5d44-49f4-a224-ca3da9566876 became leader 2026/09/15 13:03:00 12:59:20 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-7877b49576-9s7l7_00a1dc39-7548-4fe5-a378-3b148fbb83df became leader 2026/09/15 13:03:00 12:59:04 INFO Pod/kubeflow-trainer-controller-manager-7877b49576-9s7l7: AddedInterface - Add eth0 [10.130.21.51/23] from ovn-kubernetes 2026/09/15 13:03:00 12:59:04 INFO Pod/kubeflow-trainer-controller-manager-7877b49576-9s7l7: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:54ee8084549687b9db3e57589fb0e108df6480ecc39143d22f4900e51f5c6991" already present on machine 2026/09/15 13:03:00 12:59:04 INFO Pod/kubeflow-trainer-controller-manager-7877b49576-9s7l7: Created - Created container: manager 2026/09/15 13:03:00 12:59:04 INFO Pod/kubeflow-trainer-controller-manager-7877b49576-9s7l7: Started - Started container manager 2026/09/15 13:03:00 12:59:03 INFO Pod/kubeflow-trainer-controller-manager-7877b49576-9s7l7: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-7877b49576-9s7l7 to ip-10-0-1-61.us-east-2.compute.internal 2026/09/15 13:03:00 12:59:03 INFO ReplicaSet/kubeflow-trainer-controller-manager-7877b49576: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-7877b49576-9s7l7 2026/09/15 13:03:00 12:59:03 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-7877b49576 from 0 to 1 2026/09/15 13:03:00 12:58:59 INFO Pod/trainer-operator-controller-manager-7c67bcdbb4-pc5pw: AddedInterface - Add eth0 [10.129.71.91/23] from ovn-kubernetes 2026/09/15 13:03:00 12:58:59 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: AddedInterface - Add eth0 [10.129.71.92/23] from ovn-kubernetes 2026/09/15 13:03:00 12:58:59 INFO Pod/trainer-operator-controller-manager-7c67bcdbb4-pc5pw: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:f31b4a44e67a1607e5dd6112832567e894ea0512d9b21a53577a41f5896c77cc" already present on machine 2026/09/15 13:03:00 12:58:59 INFO Pod/trainer-operator-controller-manager-7c67bcdbb4-pc5pw: Created - Created container: manager 2026/09/15 13:03:00 12:58:59 INFO Pod/trainer-operator-controller-manager-7c67bcdbb4-pc5pw: Started - Started container manager 2026/09/15 13:03:00 12:58:59 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:7caa7a001cc41b2cdb15f529cde6e0a3b3bc8784e959fb935716f9d312485abc" already present on machine 2026/09/15 13:03:00 12:58:59 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: Created - Created container: manager 2026/09/15 13:03:00 12:58:59 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: Started - Started container manager 2026/09/15 13:03:00 12:58:58 INFO Pod/kserve-module-controller-manager-858fdb6fd-zqr4b: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-858fdb6fd-zqr4b to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:58:58 INFO Pod/trainer-operator-controller-manager-7c67bcdbb4-pc5pw: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-7c67bcdbb4-pc5pw to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:58:58 INFO ReplicaSet/kserve-module-controller-manager-858fdb6fd: SuccessfulCreate - Created pod: kserve-module-controller-manager-858fdb6fd-zqr4b 2026/09/15 13:03:00 12:58:58 INFO ReplicaSet/trainer-operator-controller-manager-7c67bcdbb4: SuccessfulCreate - Created pod: trainer-operator-controller-manager-7c67bcdbb4-pc5pw 2026/09/15 13:03:00 12:58:58 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-7c67bcdbb4 from 0 to 1 2026/09/15 13:03:00 12:58:58 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-858fdb6fd from 0 to 1 2026/09/15 13:03:00 12:58:47 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:43 INFO Pod/kserve-module-controller-manager-858fdb6fd-fnlbw: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:43 INFO Pod/odh-model-controller-8754f79ff-96wvs: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:43 INFO Pod/llmisvc-controller-manager-6dffbf4d96-5kzf7: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:43 INFO Pod/model-serving-api-74db48b547-rhqvl: Killing - Stopping container server 2026/09/15 13:03:00 12:58:43 INFO Pod/kserve-controller-manager-78f748ccdf-j2tt9: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:42 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 12:58:42 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 12:58:42 WARN ServiceMonitor/trustyai-service-operator-service-monitor: InvalidConfiguration - "redhat-ods-applications/trustyai-service-operator-service-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 12:58:42 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits 2026/09/15 13:03:00 12:58:40 INFO Lease/b7e9931f.trustyai.opendatahub.io: LeaderElection - trustyai-service-operator-controller-manager-86644cfc8b-nb8xm_4138ae4c-538e-4758-bf55-68b4deb71241 became leader 2026/09/15 13:03:00 12:58:24 INFO Pod/model-registry-operator-controller-manager-6958f49b4-q2wnv: BackOff - Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646" 2026/09/15 13:03:00 12:58:24 WARN Pod/model-registry-operator-controller-manager-6958f49b4-q2wnv: Failed - Error: ImagePullBackOff 2026/09/15 13:03:00 12:58:22 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: Started - Started container manager 2026/09/15 13:03:00 12:58:22 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: Created - Created container: manager 2026/09/15 13:03:00 12:58:22 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: Pulled - Container image "quay.io/rhoai/odh-trustyai-service-operator-rhel9@sha256:8bd0221526c5b263c9e829b4f81af49920958b5297855c6342956fdd81e84e10" already present on machine 2026/09/15 13:03:00 12:58:22 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: AddedInterface - Add eth0 [10.129.71.90/23] from ovn-kubernetes 2026/09/15 13:03:00 12:58:21 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm: Scheduled - Successfully assigned redhat-ods-applications/trustyai-service-operator-controller-manager-86644cfc8b-nb8xm to ip-10-0-1-132.us-east-2.compute.internal 2026/09/15 13:03:00 12:58:21 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-55j2c: Killing - Stopping container manager 2026/09/15 13:03:00 12:58:21 INFO Deployment/trustyai-service-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trustyai-service-operator-controller-manager-86644cfc8b from 0 to 1 2026/09/15 13:03:00 12:58:21 INFO ReplicaSet/trustyai-service-operator-controller-manager-86644cfc8b: SuccessfulCreate - Created pod: trustyai-service-operator-controller-manager-86644cfc8b-nb8xm 2026/09/15 13:03:00 12:58:21 INFO Pod/trustyai-service-operator-controller-manager-86644cfc8b-z52h5: Killing - Stopping container manager 2026/09/15 13:03:00 === RESOURCE QUOTAS === 2026/09/15 13:03:00 No resource quotas found in monitored namespaces 2026/09/15 13:03:00 Classification for components: infrastructure/image-pull (code=1001, confidence=medium) 2026/09/15 13:03:00 Evidence: container model-registry-operator-controller-manager-6958f49b4-q2wnv/manager waiting: ImagePullBackOff Back-off pulling image "registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"image-pull","error_code":1001,"evidence":["container model-registry-operator-controller-manager-6958f49b4-q2wnv/manager waiting: ImagePullBackOff Back-off pulling image \"registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/rhoai/odh-model-registry-operator-rhel9@sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown; artifact err: get manifest: build image source: reading manifest sha256:223a027cc0482282b61eff31aece6021e8baf4815bf9086866eb697b39e57646 in registry.redhat.io/rhoai/odh-model-registry-operator-rhel9: manifest unknown"],"confidence":"medium"} 2026/09/15 13:03:00 Diagnostics complete for failed test: components controller_test.go:497: Stopping: components test failed. 2026/09/15 13:03:00 Circuit breaker: failure classified as infrastructure/image-pull (code=1001) 2026/09/15 13:03:00 Circuit breaker: infrastructure/unknown failure recorded (3/3) 2026/09/15 13:03:00 Circuit breaker: threshold reached (3 failures), running confirming health check... 2026/09/15 13:03:01 CIRCUIT BREAKER TRIPPED: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [unhealthy nodes: ip-10-0-1-158.us-east-2.compute.internal (Ready=False)] Deletion Policy: Never. Skipping deletion tests. 2026/09/15 13:03:01 === CIRCUIT BREAKER SUMMARY === 2026/09/15 13:03:01 Tripped 1 time(s) 2026/09/15 13:03:01 Reason: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [unhealthy nodes: ip-10-0-1-158.us-east-2.compute.internal (Ready=False)] 2026/09/15 13:03:01 =============================== controller_test.go:430: Circuit breaker tripped: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [unhealthy nodes: ip-10-0-1-158.us-east-2.compute.internal (Ready=False)]]]>