{"level":"info","ts":"2026-04-16T21:04:58Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:04:58Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:04:59Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"0c10466e-9e6d-4d63-b5cf-941cd0214a72","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"50912809-60d3-4dd0-a8d2-380bbd8c4792","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-04-16T21:04:59Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"454b9e38-b762-4c57-8220-d945c3f76ad7","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:04:59Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:00Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"5d05ff60-6f1b-4c55-8421-0274d54ee812","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"2db2857e-3f5a-448a-9eb7-14645f0cd325","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:00Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"b1050fc3-8236-4657-9e59-1583b1b6cbc1","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5b0d0302-59af-4d04-a057-37bc5ce47214","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"638a9e69-ce2a-432e-bee6-94560f69c5c7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:02Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"af7edda3-aa42-4e39-8bec-adc2f57e0b86","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25a5297-feb6-4666-ba31-5f0b1b414743","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:04Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"6fc25243-8d83-43cd-8a0e-79686ef1a9ab","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f769f6e9-d7f9-4fce-bec5-2a4710ea2528","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"cd55407e-0ae4-4bf4-9bb5-203b8f2f45aa"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"cd55407e-0ae4-4bf4-9bb5-203b8f2f45aa"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"cd55407e-0ae4-4bf4-9bb5-203b8f2f45aa","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:08Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"cd55407e-0ae4-4bf4-9bb5-203b8f2f45aa","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff980f8b-8cce-44aa-bbcb-98fa38d83f0b"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff980f8b-8cce-44aa-bbcb-98fa38d83f0b"} {"level":"info","ts":"2026-04-16T21:05:08Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff980f8b-8cce-44aa-bbcb-98fa38d83f0b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:08Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ff980f8b-8cce-44aa-bbcb-98fa38d83f0b","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f214a360-a573-4c38-9f28-f3d1d0a19740"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f214a360-a573-4c38-9f28-f3d1d0a19740"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f214a360-a573-4c38-9f28-f3d1d0a19740","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:09Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"f214a360-a573-4c38-9f28-f3d1d0a19740","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1"} {"level":"info","ts":"2026-04-16T21:05:09Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:10Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"217e1124-820a-4fd4-a592-3efa3d5129f1","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:10Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"78056e18-b92a-4545-89df-9d0b4d30c998","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bc87970b-10f5-4ebe-913e-6c6a953dac2e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"347f23ce-ade4-47be-9fdc-fbaea70306ee"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"347f23ce-ade4-47be-9fdc-fbaea70306ee"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"347f23ce-ade4-47be-9fdc-fbaea70306ee","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:18Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"347f23ce-ade4-47be-9fdc-fbaea70306ee","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"94a174b5-53d4-46a6-a968-a35fd617c76c"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"94a174b5-53d4-46a6-a968-a35fd617c76c"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"94a174b5-53d4-46a6-a968-a35fd617c76c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:18Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"94a174b5-53d4-46a6-a968-a35fd617c76c","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"reconcile","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"apply","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/modelcontroller.initialize"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:18Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"0176770e-2a8e-4c28-bca5-67dd5852677a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"1a9b1e99-6894-43dc-8e9d-634754a847c0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5b9c1c72-bbfe-403a-9fec-11817511dfb3"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5b9c1c72-bbfe-403a-9fec-11817511dfb3"} {"level":"info","ts":"2026-04-16T21:05:19Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5b9c1c72-bbfe-403a-9fec-11817511dfb3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:19Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5b9c1c72-bbfe-403a-9fec-11817511dfb3","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:20Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:21Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ab4e742d-e987-444f-99c9-7b098bdfafe5","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:cc693b76-1ab8-4fde-ae35-f7bd4630448f]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:21Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"715f74ca-b512-47d7-a6c9-55eede2310d8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"37a41230-9fc6-4593-99d8-a1e487ad39cd"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"37a41230-9fc6-4593-99d8-a1e487ad39cd"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"37a41230-9fc6-4593-99d8-a1e487ad39cd","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:22Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"37a41230-9fc6-4593-99d8-a1e487ad39cd","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"59b51e32-6c0e-421d-8854-f3d29a768f19"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"59b51e32-6c0e-421d-8854-f3d29a768f19"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"59b51e32-6c0e-421d-8854-f3d29a768f19","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:22Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"59b51e32-6c0e-421d-8854-f3d29a768f19","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9646be88-41d9-4b4d-9f2e-817cf9281818"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9646be88-41d9-4b4d-9f2e-817cf9281818"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9646be88-41d9-4b4d-9f2e-817cf9281818","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"error","ts":"2026-04-16T21:05:22Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"9646be88-41d9-4b4d-9f2e-817cf9281818","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e980c370-a816-402d-95ad-a1cb3a0ecc4d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:22Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure.detectAndSetIngressMode","msg":"Detected ingress mode from existing Gateway Service","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","mode":"OcpRoute"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:22Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} W0416 21:05:25.217414 1 warnings.go:70] v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c9e6a984-7fed-412c-ad38-a4418e3889bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e2adf1f8-f118-48ff-bccf-9f5f7362b50f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:25Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e1cc8862-e03e-4135-99af-384fc6f0642e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"f5526883-d76a-4f21-a8cf-c94bcba81d99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6d7eeb69-cf97-471e-ac6d-07c225ed007d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:26Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c1a6bcff-53ae-4492-87ce-e098e72396c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:28Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b2c35173-1e8f-4b36-a5e3-2faf48c7ca51"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b2c35173-1e8f-4b36-a5e3-2faf48c7ca51"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b2c35173-1e8f-4b36-a5e3-2faf48c7ca51","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:29Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b2c35173-1e8f-4b36-a5e3-2faf48c7ca51","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"bc89da39-06e3-4783-9856-53366666cd2c","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:29Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"84e404a7-e96e-444f-9d87-e10f6cab6bf0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e93c957c-68a8-48d3-bdce-bcddd968092b"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e93c957c-68a8-48d3-bdce-bcddd968092b"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e93c957c-68a8-48d3-bdce-bcddd968092b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:05:30Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"e93c957c-68a8-48d3-bdce-bcddd968092b","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"ec759b89-6a1f-47c3-854d-1303df27b465","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:30Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"a9182b1b-1ed0-458a-bc93-1c555ce7beb1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"6fb8e66d-2593-4188-ada3-119459fd50db"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"da37c9ce-cac5-4208-ae13-ae7f1e259ca4"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"64c296ef-cb68-4322-8da4-306ad09f5fbf"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-04-16T21:05:41Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8d52ac0e-6487-40a7-a7fb-18aef6985ed1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:42Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"5295a8e7-1e44-4498-91c4-a366bd0d33ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:05:56Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4266313-5cff-4d89-88ff-3bafbb9cde08","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:07:39Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"79e6472a-5bac-468b-a48a-78b356f84a23"} {"level":"info","ts":"2026-04-16T21:07:39Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"79e6472a-5bac-468b-a48a-78b356f84a23"} {"level":"info","ts":"2026-04-16T21:07:39Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"79e6472a-5bac-468b-a48a-78b356f84a23","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:07:39Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"79e6472a-5bac-468b-a48a-78b356f84a23","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:07:59Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"1955c0e6-cf35-4db6-b666-b218f7945160"} {"level":"info","ts":"2026-04-16T21:07:59Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"1955c0e6-cf35-4db6-b666-b218f7945160"} {"level":"info","ts":"2026-04-16T21:07:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"1955c0e6-cf35-4db6-b666-b218f7945160","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:07:59Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"1955c0e6-cf35-4db6-b666-b218f7945160","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-04-16T21:08:03Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"a3ca11d4-f9be-4e38-bcd6-d20064665494"} {"level":"info","ts":"2026-04-16T21:08:03Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"31e1facf-8e53-4600-8904-c4e23ea8459e"} {"level":"info","ts":"2026-04-16T21:08:03Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"9b88ba5c-bd07-4975-a88f-9794663d092d"} {"level":"info","ts":"2026-04-16T21:08:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"4f3860e4-fd98-4c0f-bbd6-62826016c7e9"} {"level":"info","ts":"2026-04-16T21:08:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"b618f477-eb22-4247-a070-9d00e612d3ad"} {"level":"info","ts":"2026-04-16T21:08:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"2e5c39d5-88b5-4cb3-a4b5-b92b9023321c"} {"level":"info","ts":"2026-04-16T21:10:08Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"2b838209-7102-4e6e-b755-f3a18e80931f"} {"level":"info","ts":"2026-04-16T21:10:08Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"9ae67278-3c87-4730-a57d-bedd08896692"} {"level":"info","ts":"2026-04-16T21:10:08Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"2729fb93-2d25-4703-9a27-e7d0d7466476"} {"level":"info","ts":"2026-04-16T21:10:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"501fd58c-1e90-4801-b6dc-c4d9c9ff3a87"} {"level":"info","ts":"2026-04-16T21:10:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"314fbe39-1c64-4592-b40d-4e9923df841f"} {"level":"info","ts":"2026-04-16T21:10:09Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"610ad6fb-5470-46ad-8bdb-05f509d54b3f"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"7c43be16-8ea1-4bea-809c-86c832a3e008","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:10Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"16362589-f950-4c6b-abbc-a6304911e6b0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:10:40Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"52325280-88f2-4c4a-b1c8-df2a2491b31a"} {"level":"info","ts":"2026-04-16T21:10:40Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"1aee23b9-83b8-4d81-ae2c-bd8abb0a3fa4"} {"level":"info","ts":"2026-04-16T21:10:40Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"a6c8c8b3-fb74-40a9-8fe5-1bf1035cc3a6"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-04-16T21:10:41Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fb6068ae-7a79-4021-9353-aa2a616bfb14","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-04-16T21:11:27Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"b1dc2067-5acc-4e18-b78b-5e2c765fe961"} {"level":"info","ts":"2026-04-16T21:11:27Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"7511caac-fe21-4ae9-80e6-860d69431223"} {"level":"info","ts":"2026-04-16T21:11:27Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"75574ddd-c3ef-4c41-895e-52285eb7eff2"} {"level":"info","ts":"2026-04-16T21:11:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-53a60d","reconcileID":"5fd32ec5-b1b4-4533-89f9-5bf83b5caac1"} {"level":"info","ts":"2026-04-16T21:11:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-53a60d","reconcileID":"8aced7e6-dbe5-4672-b59f-656c9979f762"} {"level":"info","ts":"2026-04-16T21:11:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-53a60d","reconcileID":"19992a67-a041-468c-bd1f-33629a9065c2"} {"level":"info","ts":"2026-04-16T21:12:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-95530d","reconcileID":"a5187df2-a670-4fcc-8b13-4502841e56f8"} {"level":"info","ts":"2026-04-16T21:12:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-95530d","reconcileID":"4f35fc0d-cdb6-41a0-b3f9-a018a05e8606"} {"level":"info","ts":"2026-04-16T21:12:28Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-95530d","reconcileID":"62556619-6710-4e39-a96e-23695e4c8610"} {"level":"info","ts":"2026-04-16T21:24:19Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"24dcfc73-68d5-4359-a0a2-0107158a1796"} {"level":"info","ts":"2026-04-16T21:24:19Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"24dcfc73-68d5-4359-a0a2-0107158a1796"} {"level":"info","ts":"2026-04-16T21:24:19Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"24dcfc73-68d5-4359-a0a2-0107158a1796","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-04-16T21:24:19Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"24dcfc73-68d5-4359-a0a2-0107158a1796","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}