{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"234b5d80-7de8-4c04-925f-f189b4dc3698","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:26Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5c15a8bf-1a0e-4921-a1d2-98c864ef5101","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:26Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4ebe91ae-0b3e-4598-8801-1515a295a9c1","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d879ac05-3bbb-4e7a-a565-32edae134bab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:26Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:27Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"545e2594-160b-469d-98a0-51d007366ba6","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:27Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"a1d77397-c7a6-40e0-a3b9-c53370dcb681","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"52fddd00-4ea0-478a-8e31-929d967ee999","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2ea6b7b0-7cb1-4858-971f-e59faf2e350b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:29Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"84834d1a-dbdb-45bf-a5d3-43b6a87dbdcb","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2c3c98e8-df5f-4410-a1ae-7f36c25697bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e5d63b53-2b53-41b6-8500-389f4f81d961","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:32Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"11c148d7-9df0-4e57-aaae-0f23aa62842d","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e5190728-17c9-4804-ab94-5606042e224f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:37Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cba76f2f-7a44-4d71-a135-bc49d255716d","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c2aa1905-324c-4b11-a0aa-d480f289e92f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9daf0df2-9bf5-4899-8658-4cb2177af1bc"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9daf0df2-9bf5-4899-8658-4cb2177af1bc"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9daf0df2-9bf5-4899-8658-4cb2177af1bc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:37Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9daf0df2-9bf5-4899-8658-4cb2177af1bc","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f3024c4c-e8e1-4928-9eee-3bc3452b1439"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f3024c4c-e8e1-4928-9eee-3bc3452b1439"}
{"level":"info","ts":"2026-04-21T14:37:37Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f3024c4c-e8e1-4928-9eee-3bc3452b1439","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:37Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f3024c4c-e8e1-4928-9eee-3bc3452b1439","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:38Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2b65fe40-97fc-4478-a850-c8db8da8b2b0"}
{"level":"info","ts":"2026-04-21T14:37:38Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2b65fe40-97fc-4478-a850-c8db8da8b2b0"}
{"level":"info","ts":"2026-04-21T14:37:38Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2b65fe40-97fc-4478-a850-c8db8da8b2b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:38Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2b65fe40-97fc-4478-a850-c8db8da8b2b0","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7ea43c7f-d3e1-47b3-a2ac-60e646b32648"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7ea43c7f-d3e1-47b3-a2ac-60e646b32648"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7ea43c7f-d3e1-47b3-a2ac-60e646b32648","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:39Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7ea43c7f-d3e1-47b3-a2ac-60e646b32648","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"deb5951b-ed77-408d-81c9-f962fd71ff38"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"deb5951b-ed77-408d-81c9-f962fd71ff38"}
{"level":"info","ts":"2026-04-21T14:37:39Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"deb5951b-ed77-408d-81c9-f962fd71ff38","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:40Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"deb5951b-ed77-408d-81c9-f962fd71ff38","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"36e44851-8ed5-442c-99da-dcc591c388f3"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"36e44851-8ed5-442c-99da-dcc591c388f3"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"36e44851-8ed5-442c-99da-dcc591c388f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:44Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"36e44851-8ed5-442c-99da-dcc591c388f3","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e9d3de2c-3f17-434d-b381-8b234e041705"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e9d3de2c-3f17-434d-b381-8b234e041705"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e9d3de2c-3f17-434d-b381-8b234e041705","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:44Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e9d3de2c-3f17-434d-b381-8b234e041705","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"74d91af9-9d38-4f3d-83f2-7b0d7ed0f8c5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"68ad5739-ae6f-4998-9d37-46546684b07b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"04220e6d-6004-4b4e-815d-e50afc85ffc0"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"04220e6d-6004-4b4e-815d-e50afc85ffc0"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"04220e6d-6004-4b4e-815d-e50afc85ffc0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:44Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"04220e6d-6004-4b4e-815d-e50afc85ffc0","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a1214e85-743f-40af-8d24-b90d5002768f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure.detectAndSetIngressMode","msg":"Detected ingress mode from existing Gateway Service","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","mode":"OcpRoute"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"reconcile","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"apply","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/modelcontroller.initialize"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"d66f467b-d035-478a-b36a-879f54942749","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:46Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6c0ea438-7e28-43a3-a9ee-a5aa8a54bbc2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"dc66ed97-7ba2-43ad-abb7-3f3f1380f59c"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"dc66ed97-7ba2-43ad-abb7-3f3f1380f59c"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"dc66ed97-7ba2-43ad-abb7-3f3f1380f59c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:47Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"dc66ed97-7ba2-43ad-abb7-3f3f1380f59c","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
W0421 14:37:47.306576       1 warnings.go:70] v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"887f17bc-490a-47be-bd9d-715315954a66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"817098d7-3448-4d9b-a9aa-5d927afd54e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:47Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00d5a384-a545-45f2-960a-bac0486c8c70","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:391f2686-98cb-4da3-803b-0c963937ecb9 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:391f2686-98cb-4da3-803b-0c963937ecb9]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"05753c96-1dab-4900-936a-b01cdc8964e9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"85080635-8529-472f-8c51-bc7ca2fc165b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a3ad96bd-b567-49ce-b838-d6c4eecb2ece","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:48Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"79f7151a-65b6-414b-bb93-4f02bf330ccc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:53Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"d441c5d6-6f27-4163-969f-01293f06b49e"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"d441c5d6-6f27-4163-969f-01293f06b49e"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"d441c5d6-6f27-4163-969f-01293f06b49e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:54Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"d441c5d6-6f27-4163-969f-01293f06b49e","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"55a11b69-79ff-42cb-96b5-ad1096223d6b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:37:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3acef923-85f6-4796-93a7-cddde6266469","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff5f83ef-d3fb-4836-aea9-f32d19f86c16"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff5f83ef-d3fb-4836-aea9-f32d19f86c16"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff5f83ef-d3fb-4836-aea9-f32d19f86c16","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:37:55Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff5f83ef-d3fb-4836-aea9-f32d19f86c16","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1d8669fd-c8e3-407f-8a43-9aa94b6ac357","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:37:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7323086c-b39e-45e2-84dd-b13b907438c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"adc036ab-ee8e-42fb-bf9e-967c634b6174","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"eca5d306-b3b0-4c68-8002-4732e79672c2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"df733d14-45fd-4524-b041-e8ddca1cb823","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:38:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"ec3685d5-3cf8-4c40-8381-17b502988755"}
{"level":"info","ts":"2026-04-21T14:38:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"ab14301a-2f05-4755-9376-f73bd03d932e"}
{"level":"info","ts":"2026-04-21T14:38:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"37b10ccf-16ba-42fe-8c07-e2b43b3e4f3e"}
{"level":"info","ts":"2026-04-21T14:40:06Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4f75d7db-556c-4d39-8337-cb8f6db1e036"}
{"level":"info","ts":"2026-04-21T14:40:06Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4f75d7db-556c-4d39-8337-cb8f6db1e036"}
{"level":"info","ts":"2026-04-21T14:40:06Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4f75d7db-556c-4d39-8337-cb8f6db1e036","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:40:06Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4f75d7db-556c-4d39-8337-cb8f6db1e036","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:40:11Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"fa0ceb10-55f9-44ff-91d6-c13c5567015a"}
{"level":"info","ts":"2026-04-21T14:40:11Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"2cb87f16-d941-46a6-a1a5-b7d560022bf6"}
{"level":"info","ts":"2026-04-21T14:40:11Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"70b86f1c-4887-41c6-bedf-929ea1c602f3"}
{"level":"info","ts":"2026-04-21T14:41:12Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"3d867bce-5ee3-4f04-82d8-ec9d173330d8"}
{"level":"info","ts":"2026-04-21T14:41:12Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"87113ec3-0ad5-4260-9fdf-942cfbd9befb"}
{"level":"info","ts":"2026-04-21T14:41:12Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"0186ec80-3a68-4619-9f5c-b2f4377618b6"}
{"level":"info","ts":"2026-04-21T14:41:15Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9393eb61-31bb-425e-91a9-1bb0e2d5081d"}
{"level":"info","ts":"2026-04-21T14:41:15Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9393eb61-31bb-425e-91a9-1bb0e2d5081d"}
{"level":"info","ts":"2026-04-21T14:41:15Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9393eb61-31bb-425e-91a9-1bb0e2d5081d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:41:15Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9393eb61-31bb-425e-91a9-1bb0e2d5081d","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:41:42Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"17592a15-7d13-4a73-8c83-403fc10351a3"}
{"level":"info","ts":"2026-04-21T14:41:42Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"17592a15-7d13-4a73-8c83-403fc10351a3"}
{"level":"info","ts":"2026-04-21T14:41:42Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"17592a15-7d13-4a73-8c83-403fc10351a3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:41:42Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"17592a15-7d13-4a73-8c83-403fc10351a3","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-04-21T14:42:18Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"4e274fe1-4a71-454a-b3b0-f8ae27877681"}
{"level":"info","ts":"2026-04-21T14:42:18Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"6cfa3cf4-e42d-491f-835f-1723941f1031"}
{"level":"info","ts":"2026-04-21T14:42:19Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"bee7e8e8-ebaf-4970-8293-7460d7858b62"}
{"level":"info","ts":"2026-04-21T14:42:19Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"320de642-0ba7-4c24-9d75-325f5b3a8621"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:42:23Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"7019d3ae-5984-4a21-ac1f-e74c94082386","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:00Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"1478f14e-d6dd-4680-a4ca-358f4e4762fa"}
{"level":"info","ts":"2026-04-21T14:44:00Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"b334e977-56d2-4136-98cf-73778ca50f96"}
{"level":"info","ts":"2026-04-21T14:44:00Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"c78851f2-63d9-42b4-a84c-01a61a2596af"}
{"level":"info","ts":"2026-04-21T14:44:01Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"8cc7086c-d263-410a-b925-dac94202dafa"}
{"level":"info","ts":"2026-04-21T14:44:01Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"6318227d-3dbc-4c8c-acd4-e8ea6877dc06"}
{"level":"info","ts":"2026-04-21T14:44:01Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"8286c608-b28a-499d-bb3e-e0f5bbacd6ac"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"580ccfa0-4323-4a88-9a09-f379e12746cf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-04-21T14:44:02Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"2aba1226-939a-4694-99ea-1a4985cfde68","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-04-21T14:46:02Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"ad0688ed-09d4-4804-9362-d974e2618097"}
{"level":"info","ts":"2026-04-21T14:46:02Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"4be35b11-12a7-48e9-a3f9-db214462d520"}
{"level":"info","ts":"2026-04-21T14:46:02Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"0e893570-c074-4d32-93d3-e206b1af30fc"}
{"level":"info","ts":"2026-04-21T14:46:48Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"542eebd0-c9ba-41e3-8f0e-b933c0a8bc0b"}
{"level":"info","ts":"2026-04-21T14:46:48Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"93bb7903-3920-452c-9f28-c629d46c9678"}
{"level":"info","ts":"2026-04-21T14:46:48Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"f5f5074f-9dda-4b69-9cd2-607819bff447"}
{"level":"info","ts":"2026-04-21T14:47:18Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-e4a6e7","reconcileID":"99b147ea-2fa7-4a06-8c8b-ff23b9622b55"}
{"level":"info","ts":"2026-04-21T14:47:18Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-e4a6e7","reconcileID":"90204930-b0f0-4929-909a-dad7c1722d31"}
{"level":"info","ts":"2026-04-21T14:47:18Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-e4a6e7","reconcileID":"74fd494f-451d-42e0-882f-6b35c6cbebd8"}
{"level":"info","ts":"2026-04-21T14:47:50Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-69626e","reconcileID":"b7183dd6-0363-4041-bb3f-1f177447272b"}
{"level":"info","ts":"2026-04-21T14:47:50Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-69626e","reconcileID":"25364f26-6e15-49ba-8c3e-830a186eee87"}
{"level":"info","ts":"2026-04-21T14:47:50Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-69626e","reconcileID":"6048089a-658c-4195-b535-5f1bc68194c6"}
{"level":"info","ts":"2026-04-21T14:56:46Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"fa2c4bda-1b2d-4d97-86c4-aa7338d3ecae"}
{"level":"info","ts":"2026-04-21T14:56:46Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"fa2c4bda-1b2d-4d97-86c4-aa7338d3ecae"}
{"level":"info","ts":"2026-04-21T14:56:46Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"fa2c4bda-1b2d-4d97-86c4-aa7338d3ecae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-04-21T14:56:46Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"fa2c4bda-1b2d-4d97-86c4-aa7338d3ecae","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}