{"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7d91813e-fd79-42c8-a9eb-937273fffa0f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:00Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"3ba8bd7b-a871-4a0f-b7a3-5e7116ef9c21","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d"} {"level":"info","ts":"2026-05-25T02:32:00Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:01Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"8b421a62-01b6-4792-baba-07b1540b505d","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"b011190d-f374-4f67-9bb7-ca210a0c904a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:01Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"00be6b14-b58c-41d8-9099-9b6dd19cc071","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:01Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"215ccfc7-caeb-48bd-9d30-653cc76ae82b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-05-25T02:32:02Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2573beb9-0f45-4657-bba2-10d4d2754883","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:02Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4c4d6ef8-6b30-47c4-8a6a-4c690b4f0fd4","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"95bde886-8dc3-4dde-bf42-dd596a64040a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:02Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:03Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"cd6642a0-e64f-41c5-93a2-328a949cd6bb","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6ec97201-20b9-46cb-8041-617f7359a2da","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:03Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:04Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d9980781-2c6f-4a3a-be85-ce08d9dbb2d9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:05Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:06Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"2e4eca84-7de5-46f3-a477-5053ffeada8b","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:06Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"f04a3df9-f077-4c32-a0bc-fa650bd2a2d5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"194cfc4c-1ce1-45a9-9def-444f52641cb9"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"194cfc4c-1ce1-45a9-9def-444f52641cb9"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"194cfc4c-1ce1-45a9-9def-444f52641cb9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:07Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"194cfc4c-1ce1-45a9-9def-444f52641cb9","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5ddcf9c4-4c2b-4c15-a03f-ead37d852aa5"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5ddcf9c4-4c2b-4c15-a03f-ead37d852aa5"} {"level":"info","ts":"2026-05-25T02:32:07Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5ddcf9c4-4c2b-4c15-a03f-ead37d852aa5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:07Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"5ddcf9c4-4c2b-4c15-a03f-ead37d852aa5","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3730c2da-f76c-4c90-8e51-856819497db1"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3730c2da-f76c-4c90-8e51-856819497db1"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3730c2da-f76c-4c90-8e51-856819497db1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:11Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3730c2da-f76c-4c90-8e51-856819497db1","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:11Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure.detectAndSetIngressMode","msg":"Detected ingress mode from existing Gateway Service","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","mode":"OcpRoute"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"8fc88fc8-fd6a-4e52-9847-c5b81bb415bf"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"8fc88fc8-fd6a-4e52-9847-c5b81bb415bf"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"8fc88fc8-fd6a-4e52-9847-c5b81bb415bf","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"error","ts":"2026-05-25T02:32:11Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"8fc88fc8-fd6a-4e52-9847-c5b81bb415bf","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:11Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"c115a049-02c8-429b-a969-2b65d505b755","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:11Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0093ca50-9899-4ea8-bc9d-71dc492d6ed8"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0093ca50-9899-4ea8-bc9d-71dc492d6ed8"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0093ca50-9899-4ea8-bc9d-71dc492d6ed8","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"86c4461d-9977-4f5a-ba38-01172608ccb0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-05-25T02:32:12Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"0093ca50-9899-4ea8-bc9d-71dc492d6ed8","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b7a14fff-66ed-4e4d-ba94-407575581043"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b7a14fff-66ed-4e4d-ba94-407575581043"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b7a14fff-66ed-4e4d-ba94-407575581043","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:12Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"b7a14fff-66ed-4e4d-ba94-407575581043","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"962202c7-beb6-480d-be04-6c8bbad343d4"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"962202c7-beb6-480d-be04-6c8bbad343d4"} {"level":"info","ts":"2026-05-25T02:32:12Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"962202c7-beb6-480d-be04-6c8bbad343d4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:12Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"962202c7-beb6-480d-be04-6c8bbad343d4","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} W0525 02:32:14.473969 1 warnings.go:70] v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"e28ebe69-8d43-4d36-a057-1f8f1fc8c1c4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"3f8a61ca-fe88-4ed9-b80b-2c6a3ed6b244","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:14Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"6a78c904-2ef9-4bb9-85fd-30c2e2df3a71","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"4c17e794-b691-4595-bf15-6fba83111599","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d9e2a0c0-a236-4ce1-b951-e55fa244c35b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"76f82db4-79d5-4e6a-8321-c594bcf152d1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:22Z","msg":"Reconciler error","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"1bf127ff-d122-4c3c-86e9-3a92f13bed99","error":"provisioning failed: failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:46d11bef-c864-40b6-9c1c-2cfb25e30e42]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"703c54b1-ef42-4406-9c69-29c48bafe840","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"reconcile","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"apply","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/modelcontroller.initialize"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"modelcontroller","controllerGroup":"components.platform.opendatahub.io","controllerKind":"ModelController","ModelController":{"name":"default-modelcontroller"},"namespace":"","name":"default-modelcontroller","reconcileID":"a90060b7-ef1e-4c94-8a9b-a1e368027eae","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"e6d8e7cc-a375-42e6-b58b-fa45b3a52f7f","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3d947c61-18bc-43e1-8a28-cbb6639b6c96"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3d947c61-18bc-43e1-8a28-cbb6639b6c96"} {"level":"info","ts":"2026-05-25T02:32:22Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3d947c61-18bc-43e1-8a28-cbb6639b6c96","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:22Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"3d947c61-18bc-43e1-8a28-cbb6639b6c96","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:29Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"ae3cca40-30a6-4724-b095-a057fa85b98b","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:30Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"c2c3683b-54ba-4fc7-94f7-14694ac28ea4"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"c2c3683b-54ba-4fc7-94f7-14694ac28ea4"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"c2c3683b-54ba-4fc7-94f7-14694ac28ea4","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:31Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"c2c3683b-54ba-4fc7-94f7-14694ac28ea4","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"d6b607ff-8247-4a83-8f16-e4e9c6123702","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Starting EventSource","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","source":"kind source: *unstructured.Unstructured"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"c044035a-f285-4518-8823-b642219e30c1","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"65b76ea5-04aa-465d-8445-0d3a13ce550a"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"65b76ea5-04aa-465d-8445-0d3a13ce550a"} {"level":"info","ts":"2026-05-25T02:32:31Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"65b76ea5-04aa-465d-8445-0d3a13ce550a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:32:31Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"65b76ea5-04aa-465d-8445-0d3a13ce550a","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"28efc167-3f73-4e4a-a237-ed0b4a342a29","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:32Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"56e68361-0faa-4958-b37c-b27a71f72822","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"reconcile","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"apply","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.initialize"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/dependency.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.removeOwnershipFromUnmanagedResources"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.cleanUpTemplatedResources"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.customizeKserveConfigMap"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kserve.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-05-25T02:32:42Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"kserve","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kserve","Kserve":{"name":"default-kserve"},"namespace":"","name":"default-kserve","reconcileID":"4515e4a8-7c65-4e3c-b36c-d473aeeee9ef","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:32:43Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"7e531cab-dd4d-4c2d-9f1e-810dae051a4a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:32:44Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"a93bf170-ffa6-4de7-b141-576bcf96dde1"} {"level":"info","ts":"2026-05-25T02:32:44Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"508b4684-7f1c-4397-884c-c35634d1f331"} {"level":"info","ts":"2026-05-25T02:32:44Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"kuadrant-system","reconcileID":"4d199ff5-4db0-4e9c-bde1-e534a253a690"} {"level":"info","ts":"2026-05-25T02:34:42Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a136b457-ff6e-4b32-aee3-248118acaae5"} {"level":"info","ts":"2026-05-25T02:34:42Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a136b457-ff6e-4b32-aee3-248118acaae5"} {"level":"info","ts":"2026-05-25T02:34:42Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a136b457-ff6e-4b32-aee3-248118acaae5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:34:42Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"a136b457-ff6e-4b32-aee3-248118acaae5","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:34:54Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"331e1a3f-a064-415b-b29b-3c0789f9e6ef"} {"level":"info","ts":"2026-05-25T02:34:54Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"e78aff18-db98-4b28-ae2d-05325a88b5fb"} {"level":"info","ts":"2026-05-25T02:34:54Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"keycloak-system","reconcileID":"526471cd-0301-4656-8c0f-9fcb6635bf94"} {"level":"info","ts":"2026-05-25T02:35:50Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"11ea88e0-479d-4558-b89c-a3091e21e277"} {"level":"info","ts":"2026-05-25T02:35:50Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"11ea88e0-479d-4558-b89c-a3091e21e277"} {"level":"info","ts":"2026-05-25T02:35:50Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"11ea88e0-479d-4558-b89c-a3091e21e277","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:35:50Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"11ea88e0-479d-4558-b89c-a3091e21e277","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:35:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"822c0cf8-6506-4a7e-81cc-beba99f6e56c"} {"level":"info","ts":"2026-05-25T02:35:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"8f63b8b3-da9e-45b1-83f1-395ba1ac0128"} {"level":"info","ts":"2026-05-25T02:35:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"models-as-a-service","reconcileID":"08fc41fb-dbee-4cf4-8d70-49d126547bbe"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ec9d219f-df06-482b-b6c4-2c8116606874"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ec9d219f-df06-482b-b6c4-2c8116606874"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ec9d219f-df06-482b-b6c4-2c8116606874","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:36:27Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ec9d219f-df06-482b-b6c4-2c8116606874","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2130c520-7a65-4811-a99f-a4f183313b15"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2130c520-7a65-4811-a99f-a4f183313b15"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2130c520-7a65-4811-a99f-a4f183313b15","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:36:27Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"2130c520-7a65-4811-a99f-a4f183313b15","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"64f0ccbd-d907-4a26-a4a9-3792e03c16b9"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"64f0ccbd-d907-4a26-a4a9-3792e03c16b9"} {"level":"info","ts":"2026-05-25T02:36:27Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"64f0ccbd-d907-4a26-a4a9-3792e03c16b9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:36:27Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"64f0ccbd-d907-4a26-a4a9-3792e03c16b9","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:36:59Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"a4bb072d-73b9-47a1-bcf5-9a67e3be68f3","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:37:15Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"d567e731-b62a-4a4d-ac43-787b9b8b4ab3"} {"level":"info","ts":"2026-05-25T02:37:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"bff8b952-b3f8-46e0-897e-b1c26ab44405"} {"level":"info","ts":"2026-05-25T02:37:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"f58b6db1-bdf1-4e8e-981e-a1a0b870889b"} {"level":"info","ts":"2026-05-25T02:37:16Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"llm","reconcileID":"31b80c1b-4f0f-4860-a36f-068c3aff8d61"} {"level":"info","ts":"2026-05-25T02:38:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"5924930d-c02b-49e8-b46a-7217f01fb25d"} {"level":"info","ts":"2026-05-25T02:38:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"714cb4a2-bd1a-433e-8c28-b42bebb52397"} {"level":"info","ts":"2026-05-25T02:38:56Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"premium-users-namespace","reconcileID":"81f42d68-d1b0-42d9-b9a2-75db8425c937"} {"level":"info","ts":"2026-05-25T02:38:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"00e81d9b-14b9-481f-96fa-efde5a35ea01"} {"level":"info","ts":"2026-05-25T02:38:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"1b80622e-c906-492e-9c99-a2435fba2762"} {"level":"info","ts":"2026-05-25T02:38:57Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"maas-admin","reconcileID":"231e923a-b993-4fea-999b-2fd09c2432bb"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"0936d17d-2213-435c-87ef-35d9393a5f59","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"reconcile","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"apply","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.initialize"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.createDefaultGroup"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/auth.managePermissions"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-05-25T02:38:58Z","msg":"Executing action","controller":"auth","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Auth","Auth":{"name":"auth"},"namespace":"","name":"auth","reconcileID":"079a000c-7e6f-473a-8207-d87af6bba40e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-05-25T02:41:04Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"b18a6b40-bd50-4492-a63d-09beb7825df8"} {"level":"info","ts":"2026-05-25T02:41:04Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"007b5d56-e1a2-4bd8-9db8-020d22d5e5bb"} {"level":"info","ts":"2026-05-25T02:41:04Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-api-unwatched-ns","reconcileID":"f9286ae8-dd30-4b3c-8dec-770c3c2ddc30"} {"level":"info","ts":"2026-05-25T02:41:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"c2a90d87-cd3f-409f-af5c-c6306486b6a6"} {"level":"info","ts":"2026-05-25T02:41:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"65f996bd-d857-4820-ba04-a0757d27e008"} {"level":"info","ts":"2026-05-25T02:41:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-unwatched-ns","reconcileID":"7da5221d-b230-454e-92a9-d5fb893eaa2c"} {"level":"info","ts":"2026-05-25T02:42:21Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-aa8b8c","reconcileID":"fbf9ecad-77bb-4e8c-b978-c6c53983ff00"} {"level":"info","ts":"2026-05-25T02:42:21Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-aa8b8c","reconcileID":"5e7429f8-5da6-4be2-b376-e4927493709a"} {"level":"info","ts":"2026-05-25T02:42:21Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-aa8b8c","reconcileID":"7b79f1ab-5ab4-4fee-8be2-5cac183670b7"} {"level":"info","ts":"2026-05-25T02:42:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-3425b3","reconcileID":"166853cb-a39f-4f67-a977-8de4ca06e20c"} {"level":"info","ts":"2026-05-25T02:42:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-3425b3","reconcileID":"3502e800-156e-4012-aec2-38e4bcc13316"} {"level":"info","ts":"2026-05-25T02:42:51Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"e2e-modelref-3425b3","reconcileID":"dced5839-0a25-4ec7-a0e3-6dfc7438ff6a"} {"level":"info","ts":"2026-05-25T02:51:22Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ae55adbe-d69d-4847-a5ed-9e72c66b07bc"} {"level":"info","ts":"2026-05-25T02:51:22Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ae55adbe-d69d-4847-a5ed-9e72c66b07bc"} {"level":"info","ts":"2026-05-25T02:51:22Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ae55adbe-d69d-4847-a5ed-9e72c66b07bc","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"error","ts":"2026-05-25T02:51:22Z","msg":"Reconciler error","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"ae55adbe-d69d-4847-a5ed-9e72c66b07bc","error":"provisioning failed: error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255"}