{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:27Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5a68ea36-335f-4878-885c-b6eca07df7e2","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:27Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:28Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d81cbca5-0cf7-443f-9b29-9282a991f70d","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"397e2526-cfdb-4be7-85dc-c6015d073208","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:28Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"69c1a924-cacf-4491-8a28-5a15398ba98b","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"35c5f3cb-f436-46bc-94d2-def4917422f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:29Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"e267671f-f135-4670-bbf7-c292e542f839","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1441235a-31f9-48bb-9314-289a9f8af4f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d0da737e-6fee-4479-acd9-2e2bb416ae75","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d32fa97d-e0cd-4e1d-8a84-823f5705d033","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:31Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33"}
{"level":"info","ts":"2026-05-21T21:27:31Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33"}
{"level":"info","ts":"2026-05-21T21:27:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:32Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0abff48a-a248-442f-be50-09b380ffde33","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"295c6454-7c62-4547-ae51-cca7e2c287f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4ab81aba-5430-444f-84e4-0fe06f31561b"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4ab81aba-5430-444f-84e4-0fe06f31561b"}
{"level":"info","ts":"2026-05-21T21:27:32Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4ab81aba-5430-444f-84e4-0fe06f31561b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:32Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"4ab81aba-5430-444f-84e4-0fe06f31561b","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:33Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9339f21f-b3bb-401b-80c6-a9d6a55b542c"}
{"level":"info","ts":"2026-05-21T21:27:33Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9339f21f-b3bb-401b-80c6-a9d6a55b542c"}
{"level":"info","ts":"2026-05-21T21:27:33Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9339f21f-b3bb-401b-80c6-a9d6a55b542c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:33Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9339f21f-b3bb-401b-80c6-a9d6a55b542c","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"14a7f4f9-19b0-4a0a-80bc-e272b81b5140"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"14a7f4f9-19b0-4a0a-80bc-e272b81b5140"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"14a7f4f9-19b0-4a0a-80bc-e272b81b5140","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:34Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"14a7f4f9-19b0-4a0a-80bc-e272b81b5140","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"984b866b-3cb7-4b75-844d-efa61dc2300e"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"984b866b-3cb7-4b75-844d-efa61dc2300e"}
{"level":"info","ts":"2026-05-21T21:27:34Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"984b866b-3cb7-4b75-844d-efa61dc2300e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:34Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"984b866b-3cb7-4b75-844d-efa61dc2300e","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"96758b2f-3a9d-42b5-9924-c59fcd0fd47b"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"96758b2f-3a9d-42b5-9924-c59fcd0fd47b"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"96758b2f-3a9d-42b5-9924-c59fcd0fd47b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:36Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"96758b2f-3a9d-42b5-9924-c59fcd0fd47b","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b0a7d616-c560-4602-b92a-b42bda73f638"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b0a7d616-c560-4602-b92a-b42bda73f638"}
{"level":"info","ts":"2026-05-21T21:27:36Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b0a7d616-c560-4602-b92a-b42bda73f638","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:36Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b0a7d616-c560-4602-b92a-b42bda73f638","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:37Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"7b5b16b4-4c36-4ad7-bd02-b90bba212e25","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:37Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2f5ab66d-a614-42ee-a64c-bde11dfd8adc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0f8d874e-cdaf-490b-9c0e-ad9772e8ee6c"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0f8d874e-cdaf-490b-9c0e-ad9772e8ee6c"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0f8d874e-cdaf-490b-9c0e-ad9772e8ee6c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:40Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0f8d874e-cdaf-490b-9c0e-ad9772e8ee6c","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"83f144d7-fc3b-40c7-a464-238371d2c936"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"83f144d7-fc3b-40c7-a464-238371d2c936"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"83f144d7-fc3b-40c7-a464-238371d2c936","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:40Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"83f144d7-fc3b-40c7-a464-238371d2c936","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:40Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"abc243ea-4204-49d8-8230-02d167c1123a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"296ca363-07b3-4a23-a944-26faf0138610","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"31d72711-8208-47e1-9ced-67806e7fa3e1"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"31d72711-8208-47e1-9ced-67806e7fa3e1"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"31d72711-8208-47e1-9ced-67806e7fa3e1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"error","ts":"2026-05-21T21:27:41Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"31d72711-8208-47e1-9ced-67806e7fa3e1","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"95ce113a-3d68-4494-8fa7-52356b5a79db","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure.detectAndSetIngressMode","msg":"Detected ingress mode from existing Gateway Service","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","mode":"OcpRoute"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"reconcile","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"apply","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/modelcontroller.initialize"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"c0812277-a85e-45ba-84f7-44c86933e7e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"464afbf7-6ade-43f0-92bc-f9f93b3a1548","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"13cd6527-75e5-4c84-85c6-581e391a8bdc"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"13cd6527-75e5-4c84-85c6-581e391a8bdc"}
{"level":"info","ts":"2026-05-21T21:27:43Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"13cd6527-75e5-4c84-85c6-581e391a8bdc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:43Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"13cd6527-75e5-4c84-85c6-581e391a8bdc","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
W0521 21:27:44.232966       1 warnings.go:70] v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f4cb049c-8f15-404e-b23c-90e5b41acd5f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"04276ac0-8433-4c3d-bf96-ab640ee72641","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e44c7fa8-6929-48f2-8847-8813e3901d87","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:44Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"21d865b4-3657-4ccc-9988-393d223cdf30","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:47Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af81bd74-97bd-4db7-8da5-b24daf735224","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:31673a4c-1c7a-45e9-98fe-ffc4b50bac6e]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n  echo \"Trying to infer RoCE configs ... \"\n  grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n  grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n  cat /proc/driver/nvidia/params\n\n  KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n  echo \"[Infer RoCE] Discovering active HCAs ...\"\n  active_hcas=()\n  # Loop through all mlx5 devices found in sysfs\n  for hca_dir in /sys/class/infiniband/mlx5_*; do\n      # Ensure it's a directory before proceeding\n      if [ -d \"$hca_dir\" ]; then\n          hca_name=$(basename \"$hca_dir\")\n          port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n          type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n          echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n          if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n              echo \"[Infer RoCE] Found active HCA: $hca_name\"\n              active_hcas+=(\"$hca_name\")\n          else\n              echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n          fi\n      fi\n  done\n\n  ucx_hcas=()\n  for hca in \"${active_hcas[@]}\"; do\n    ucx_hcas+=(\"${hca}:1\")\n  done\n\n  # Check if we found any active HCAs\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      # Join the array elements with a comma\n      hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n      echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n      export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n      export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n      export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n      echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n      echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n  else\n      echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n  fi\n\n  if [ ${#active_hcas[@]} -gt 0 ]; then\n      echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n      # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n      declare -A gid_index_count\n      declare -A hca_gid_index\n\n      for hca_name in \"${active_hcas[@]}\"; do\n          echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n          # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n          for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n              if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n                  idx=$(basename \"$tpath\")\n                  gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n                  # Check for IPv4 GID (contains ffff:)\n                  if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n                      gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n                      echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n                      hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n                      gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n                      break  # Use first found IPv4 GID per HCA\n                  fi\n              fi\n          done\n      done\n\n      # Find the most common GID index (most likely to be consistent across nodes)\n      best_gid_index=\"\"\n      max_count=0\n      for idx in \"${!gid_index_count[@]}\"; do\n          count=${gid_index_count[\"${idx}\"]}\n          echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n          if [ $count -gt $max_count ]; then\n              max_count=$count\n              best_gid_index=\"$idx\"\n          fi\n      done\n\n      # Use deterministic fallback if counts are equal - prefer lower index number  \n      if [ ${#gid_index_count[@]} -gt 1 ]; then\n          echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n          # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n          if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n              best_gid_index=\"3\"\n              echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n          fi\n      fi\n\n      # Check if GID_INDEX is already set via environment variables\n      if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n          echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n          echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n      elif [ -n \"$best_gid_index\" ]; then\n          echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n          export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n          export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n          export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n          echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n      else\n          echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n      fi\n  else\n      echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n  fi\nfi\n\neval \"vllm serve /mnt/models \\\n  --served-model-name \"{{ .Spec.Model.Name }}\" \\\n  --port 8001 \\\n  ${VLLM_ADDITIONAL_ARGS} \\\n  --enable-ssl-refresh \\\n  --ssl-certfile /var/run/kserve/tls/tls.crt \\\n  --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:47Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:48Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8f0c3c6a-edd4-43c7-bb50-bf12cab7521b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a406299c-ca61-4eeb-a10c-f9b4f005841b"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a406299c-ca61-4eeb-a10c-f9b4f005841b"}
{"level":"info","ts":"2026-05-21T21:27:54Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a406299c-ca61-4eeb-a10c-f9b4f005841b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:54Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a406299c-ca61-4eeb-a10c-f9b4f005841b","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ecf119e5-fb91-4892-b6c8-39245a938482","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"3735f4be-b26f-49a9-bd3c-d5a430495127","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2e6b4ec8-6b0a-4f6c-9599-0b4b0fa8f0e3"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2e6b4ec8-6b0a-4f6c-9599-0b4b0fa8f0e3"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2e6b4ec8-6b0a-4f6c-9599-0b4b0fa8f0e3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:27:55Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2e6b4ec8-6b0a-4f6c-9599-0b4b0fa8f0e3","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d23c3b3d-160a-4a4c-913e-164111de865a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:27:55Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"8d4efb56-2e96-4469-89b9-57989d65806b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:06Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"0777dd62-d84c-4103-b7a5-47f9d36f3053"}
{"level":"info","ts":"2026-05-21T21:28:06Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"d8e54115-8b63-4e0a-a51a-60bba4b768e0"}
{"level":"info","ts":"2026-05-21T21:28:06Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"ffb58d54-a58f-4ce8-a853-a82bb21161c6"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bda26942-3f60-430e-8a5a-ef5acd3c3cde","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b02d116a-12aa-45df-b32a-f7484ffdd0b8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:08Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"67e9cfb9-2d54-4a3a-ba9f-86b71461e5c9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:28:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d1527813-4e05-4f9c-8631-33d7a798c267","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:30:03Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68be449e-a7b7-4146-b5bb-ce70423d36ba"}
{"level":"info","ts":"2026-05-21T21:30:03Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68be449e-a7b7-4146-b5bb-ce70423d36ba"}
{"level":"info","ts":"2026-05-21T21:30:03Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68be449e-a7b7-4146-b5bb-ce70423d36ba","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:30:03Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68be449e-a7b7-4146-b5bb-ce70423d36ba","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:30:25Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7cbb6bb7-1b96-44d8-8d3f-de9f29649df1"}
{"level":"info","ts":"2026-05-21T21:30:25Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7cbb6bb7-1b96-44d8-8d3f-de9f29649df1"}
{"level":"info","ts":"2026-05-21T21:30:25Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7cbb6bb7-1b96-44d8-8d3f-de9f29649df1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:30:25Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7cbb6bb7-1b96-44d8-8d3f-de9f29649df1","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:30:31Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"ed43e4e6-b7c2-4299-b6e7-b0d99c5be560"}
{"level":"info","ts":"2026-05-21T21:30:31Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"bfaad4f0-5cc9-4321-87bc-516ccac1602c"}
{"level":"info","ts":"2026-05-21T21:30:31Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"378e96a5-afb0-4495-a1fa-696bbc75d695"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2edaebbc-5b1a-4940-bd77-17783074e478"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2edaebbc-5b1a-4940-bd77-17783074e478"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2edaebbc-5b1a-4940-bd77-17783074e478","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:31:00Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2edaebbc-5b1a-4940-bd77-17783074e478","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"228807a2-4a3d-42b7-a5d6-b53f17aff103"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"228807a2-4a3d-42b7-a5d6-b53f17aff103"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"228807a2-4a3d-42b7-a5d6-b53f17aff103","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:31:00Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"228807a2-4a3d-42b7-a5d6-b53f17aff103","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"20c10246-5696-4267-a772-e99ed433c8e7"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"20c10246-5696-4267-a772-e99ed433c8e7"}
{"level":"info","ts":"2026-05-21T21:31:00Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"20c10246-5696-4267-a772-e99ed433c8e7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:31:00Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"20c10246-5696-4267-a772-e99ed433c8e7","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}
{"level":"info","ts":"2026-05-21T21:31:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"5426be12-1ca3-4cfb-9130-79c90e48fe82"}
{"level":"info","ts":"2026-05-21T21:31:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"caf7b3cc-c4ae-420a-8e4a-11aba6437e78"}
{"level":"info","ts":"2026-05-21T21:31:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"c9852465-868d-48a1-b9cf-094ce31a747d"}
{"level":"info","ts":"2026-05-21T21:31:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"a1e1984e-d39b-4ffc-8358-64ab9cb59aaa"}
{"level":"info","ts":"2026-05-21T21:32:24Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"4063af3e-9204-45b3-b8a9-c33741a0c6c6"}
{"level":"info","ts":"2026-05-21T21:32:24Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"cb2a98e7-dba5-4ebe-9b2b-13a7a6921fb4"}
{"level":"info","ts":"2026-05-21T21:32:24Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"c5e25a35-2e55-4436-9d47-ec3b89a709af"}
{"level":"info","ts":"2026-05-21T21:32:25Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"bbf56989-2741-4d3f-9f06-bd64015b9695"}
{"level":"info","ts":"2026-05-21T21:32:25Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"6f13e711-a43a-4383-90a2-283c48da588e"}
{"level":"info","ts":"2026-05-21T21:32:26Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"ea543a11-a68d-4660-8337-52dcef27eb38"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"745106d6-23a4-4f7d-9cfe-3f5e3a3c0b5d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:32:27Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"e7940468-f220-4ae1-bccf-a4e7c47b5b25","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"}
{"level":"info","ts":"2026-05-21T21:33:07Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4f7a37b5-81c0-4ef5-8dd1-4cc42fcc6f90","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"}
{"level":"info","ts":"2026-05-21T21:34:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"5c4bd185-0eaf-44d4-996f-ae0813dda99e"}
{"level":"info","ts":"2026-05-21T21:34:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"214c3af5-77dd-4c9b-b3c6-e6b7df53d450"}
{"level":"info","ts":"2026-05-21T21:34:29Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"39249591-f176-4c7c-95cc-07bd1c171617"}
{"level":"info","ts":"2026-05-21T21:34:29Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"c24a9096-1a5e-4df7-9481-d77a06ec07f3"}
{"level":"info","ts":"2026-05-21T21:35:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"63866e8d-61bf-4f53-bd5d-3bf64f3fa0b9"}
{"level":"info","ts":"2026-05-21T21:35:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"197ccb80-9930-40c0-967e-54da46bc91b9"}
{"level":"info","ts":"2026-05-21T21:35:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"4aef4006-cdaf-4e2f-88f8-b18f0015a4de"}
{"level":"info","ts":"2026-05-21T21:35:46Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-a06827","reconcileID":"ec91104d-27ad-4ff4-9faa-f3c966a21017"}
{"level":"info","ts":"2026-05-21T21:35:46Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-a06827","reconcileID":"7b925f64-068a-4ac3-a947-b9f15161761a"}
{"level":"info","ts":"2026-05-21T21:35:46Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-a06827","reconcileID":"3d7cdce1-7a9e-4eae-95fa-a89393a7524e"}
{"level":"info","ts":"2026-05-21T21:36:17Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-72e0b0","reconcileID":"ecc26b90-4bad-4d78-98cd-51c8ef2e92f9"}
{"level":"info","ts":"2026-05-21T21:36:17Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-72e0b0","reconcileID":"6fe91dcd-4517-4528-81d9-f15a824de7ec"}
{"level":"info","ts":"2026-05-21T21:36:17Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-72e0b0","reconcileID":"c84a1367-4fbc-438e-93e9-78ea9a869bc9"}
{"level":"info","ts":"2026-05-21T21:46:43Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68af0f9b-81c7-4459-bd5b-7c9d097f098a"}
{"level":"info","ts":"2026-05-21T21:46:43Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68af0f9b-81c7-4459-bd5b-7c9d097f098a"}
{"level":"info","ts":"2026-05-21T21:46:43Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68af0f9b-81c7-4459-bd5b-7c9d097f098a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"}
{"level":"error","ts":"2026-05-21T21:46:43Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"68af0f9b-81c7-4459-bd5b-7c9d097f098a","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}