--- apiVersion: v1 items: - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:27Z" involvedObject: kind: CertificateSigningRequest name: csr-6kkwg kind: Event lastTimestamp: "2026-05-28T14:53:27Z" message: CSR "csr-6kkwg" has been approved metadata: creationTimestamp: "2026-05-28T14:53:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:53:27Z" name: csr-6kkwg.18b3c262b8a46a88 namespace: default resourceVersion: "6545" uid: 85b289b7-47dc-4cd2-afac-92d9ded068f3 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:33Z" involvedObject: kind: CertificateSigningRequest name: csr-cxv4t kind: Event lastTimestamp: "2026-05-28T14:53:33Z" message: CSR "csr-cxv4t" has been approved metadata: creationTimestamp: "2026-05-28T14:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:53:33Z" name: csr-cxv4t.18b3c26434129d2b namespace: default resourceVersion: "6643" uid: fbb62968-5661-4bf3-8eaa-04e06b0ad3b9 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:52:59Z" involvedObject: kind: CertificateSigningRequest name: csr-kzvgd kind: Event lastTimestamp: "2026-05-28T14:52:59Z" message: CSR "csr-kzvgd" has been approved metadata: creationTimestamp: "2026-05-28T14:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:52:59Z" name: csr-kzvgd.18b3c25c50f540c6 namespace: default resourceVersion: "5703" uid: 5a2c16ac-b3da-4ace-9da8-66b8a34ce07d reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:33Z" involvedObject: kind: CertificateSigningRequest name: csr-mjrzp kind: Event lastTimestamp: "2026-05-28T14:53:33Z" message: CSR "csr-mjrzp" has been approved metadata: creationTimestamp: "2026-05-28T14:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:53:33Z" name: csr-mjrzp.18b3c2644702f12d namespace: default resourceVersion: "6648" uid: 439a5fc5-2d94-4ce7-9872-291bbc8af0ab reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:05Z" involvedObject: kind: CertificateSigningRequest name: csr-nwxc2 kind: Event lastTimestamp: "2026-05-28T14:53:05Z" message: CSR "csr-nwxc2" has been approved metadata: creationTimestamp: "2026-05-28T14:53:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:53:05Z" name: csr-nwxc2.18b3c25dcb1662a7 namespace: default resourceVersion: "6131" uid: 89525098-bc61-48f9-a9e6-e481291cd8fd reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:26Z" involvedObject: kind: CertificateSigningRequest name: csr-wq2xp kind: Event lastTimestamp: "2026-05-28T14:53:26Z" message: CSR "csr-wq2xp" has been approved metadata: creationTimestamp: "2026-05-28T14:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T14:53:26Z" name: csr-wq2xp.18b3c262b428781d namespace: default resourceVersion: "6539" uid: c2b7802f-f34b-4c9e-bb0c-32f8ea7bd0f8 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-28T15:00:35Z" involvedObject: apiVersion: datasciencecluster.opendatahub.io/v2 kind: DataScienceCluster name: default-dsc resourceVersion: "14703" uid: c529c187-8426-425e-abf3-63781fe2c88c kind: Event lastTimestamp: "2026-05-28T15:00:37Z" message: 'failure deploying resource {map[apiVersion:components.platform.opendatahub.io/v1alpha1 kind:Kserve metadata:map[annotations:map[component.opendatahub.io/management-state:Managed platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-dsc platform.opendatahub.io/instance.uid:c529c187-8426-425e-abf3-63781fe2c88c platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1] creationTimestamp: labels:map[platform.opendatahub.io/part-of:datasciencecluster] name:default-kserve ownerReferences:[map[apiVersion:datasciencecluster.opendatahub.io/v2 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:DataScienceCluster name:default-dsc uid:c529c187-8426-425e-abf3-63781fe2c88c]]] spec:map[modelsAsService:map[managementState:Removed] nim:map[managementState:Managed] rawDeploymentServiceConfig:Headed] status:map[]]}: apply failed components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve default-kserve: kserves.components.platform.opendatahub.io "default-kserve" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion datasciencecluster.opendatahub.io/v2 Kind DataScienceCluster: no matches for kind "DataScienceCluster" in version "datasciencecluster.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-28T15:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T15:00:37Z" name: default-dsc.18b3c2c66a9f925a namespace: default resourceVersion: "14769" uid: 442d9cee-8af5-4216-96e2-94e49286c229 reason: ProvisioningError reportingComponent: datasciencecluster reportingInstance: "" source: component: datasciencecluster type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-28T15:00:34Z" involvedObject: apiVersion: dscinitialization.opendatahub.io/v2 kind: DSCInitialization name: default-dsci resourceVersion: "14570" uid: 641ca7e2-cc07-46ec-b8c0-a1b287aa9459 kind: Event lastTimestamp: "2026-05-28T15:00:37Z" message: 'failed to create operator resources for instance default-dsci: unable to patch networking.k8s.io/v1, Kind=NetworkPolicy opendatahub/opendatahub: networkpolicies.networking.k8s.io "opendatahub" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion dscinitialization.opendatahub.io/v2 Kind DSCInitialization: no matches for kind "DSCInitialization" in version "dscinitialization.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-28T15:00:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T15:00:37Z" name: default-dsci.18b3c2c658cdd25f namespace: default resourceVersion: "14766" uid: a0ad2c0e-7e20-49af-9cd3-3eeb1865f06e reason: DSCInitializationReconcileError reportingComponent: dscinitialization-controller reportingInstance: "" source: component: dscinitialization-controller type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-28T15:00:41Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14878" uid: 3479f838-1d8c-4d62-8689-b2bf29f2ea68 kind: Event lastTimestamp: "2026-05-28T15:00:48Z" message: 'failed to lookup object openshift-ingress/data-science-tls-rule: no matches for kind "DestinationRule" in version "networking.istio.io/v1"' metadata: creationTimestamp: "2026-05-28T15:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T15:00:48Z" name: default-gateway.18b3c2c7e04ddbc6 namespace: default resourceVersion: "15092" uid: d29a6ac4-50a5-499f-9b66-f6a75d2b6a73 reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-28T15:00:49Z" involvedObject: apiVersion: components.platform.opendatahub.io/v1alpha1 kind: Kserve name: default-kserve resourceVersion: "15190" uid: 70f7e64e-c3b8-4116-ab04-d3bbf64ed485 kind: Event lastTimestamp: "2026-05-28T15:01:17Z" message: "failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:70f7e64e-c3b8-4116-ab04-d3bbf64ed485 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:70f7e64e-c3b8-4116-ab04-d3bbf64ed485]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n \ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n \ echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n \ echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n \ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n \ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n \ if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n \ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n \ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n \ # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n \ gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n \ fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n \ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n \ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n \ fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n \ echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n \ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n \ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n \ export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n \ echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"" metadata: creationTimestamp: "2026-05-28T15:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T15:01:17Z" name: default-kserve.18b3c2c9db5ca251 namespace: default resourceVersion: "16640" uid: 014e9eea-3d9c-4ad4-8654-452538aa8752 reason: ProvisioningError reportingComponent: kserve reportingInstance: "" source: component: kserve type: Warning - apiVersion: v1 count: 34 eventTime: null firstTimestamp: "2026-05-28T15:00:40Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Monitoring name: default-monitoring resourceVersion: "14789" uid: 068c2dee-96dd-43ee-a8e2-563227700f41 kind: Event lastTimestamp: "2026-05-28T15:18:10Z" message: 'error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache' metadata: creationTimestamp: "2026-05-28T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T15:18:10Z" name: default-monitoring.18b3c2c79de83aee namespace: default resourceVersion: "36850" uid: efeac0f1-3158-4f2a-bcd7-8a6c5a1a40c1 reason: ProvisioningError reportingComponent: monitoring reportingInstance: "" source: component: monitoring type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: kind: Node name: ip-10-0-128-181.ec2.internal uid: ip-10-0-128-181.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: 'Node ip-10-0-128-181.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-128-181.ec2.internal.18b3c25cae658d00 namespace: default resourceVersion: "5958" uid: 327624e8-822f-460b-9d25-a9008ad1b612 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-128-181.ec2.internal source: component: kubelet host: ip-10-0-128-181.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: kind: Node name: ip-10-0-128-181.ec2.internal uid: ip-10-0-128-181.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: 'Node ip-10-0-128-181.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-128-181.ec2.internal.18b3c25cae65ce83 namespace: default resourceVersion: "5959" uid: 82e67076-8587-4301-8463-6b929789d898 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-128-181.ec2.internal source: component: kubelet host: ip-10-0-128-181.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: kind: Node name: ip-10-0-128-181.ec2.internal uid: ip-10-0-128-181.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: 'Node ip-10-0-128-181.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-128-181.ec2.internal.18b3c25cae65f345 namespace: default resourceVersion: "5960" uid: 9a7bae8d-aece-4969-9097-768f3ab423bc reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-128-181.ec2.internal source: component: kubelet host: ip-10-0-128-181.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: kind: Node name: ip-10-0-128-181.ec2.internal uid: ip-10-0-128-181.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-128-181.ec2.internal.18b3c25cb0f884c1 namespace: default resourceVersion: "5859" uid: da8dadad-6f2c-41a1-b1b0-33a8659c5035 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-128-181.ec2.internal source: component: kubelet host: ip-10-0-128-181.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-181.ec2.internal resourceVersion: "5863" uid: e47c1044-c36b-43bc-9e9a-cf4a3752ddb4 kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-128-181.ec2.internal.18b3c25cc5740787 namespace: default resourceVersion: "5965" uid: 34b67f7e-5ec0-41cb-bd5b-38d40785b33c reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:06Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-181.ec2.internal uid: e47c1044-c36b-43bc-9e9a-cf4a3752ddb4 kind: Event lastTimestamp: "2026-05-28T14:53:06Z" message: 'Node ip-10-0-128-181.ec2.internal event: Registered Node ip-10-0-128-181.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:53:06Z" name: ip-10-0-128-181.ec2.internal.18b3c25ddb5482c5 namespace: default resourceVersion: "6139" uid: ba2e020c-f3d6-44c0-9eb2-8567fb26f020 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:28Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-181.ec2.internal resourceVersion: "6569" uid: e47c1044-c36b-43bc-9e9a-cf4a3752ddb4 kind: Event lastTimestamp: "2026-05-28T14:53:28Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-128-181.ec2.internal, error getting gateway config for node ip-10-0-128-181.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-128-181.ec2.internal", failed to update chassis to local for local node ip-10-0-128-181.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-128-181.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-128-181.ec2.internal]' metadata: creationTimestamp: "2026-05-28T14:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-128-181 operation: Update time: "2026-05-28T14:53:28Z" name: ip-10-0-128-181.ec2.internal.18b3c26313cf5100 namespace: default resourceVersion: "6578" uid: 2b30ce39-9df0-41f4-8353-1319db1853eb reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:34Z" involvedObject: kind: Node name: ip-10-0-128-181.ec2.internal uid: ip-10-0-128-181.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:34Z" message: 'Node ip-10-0-128-181.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T14:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:34Z" name: ip-10-0-128-181.ec2.internal.18b3c2648cf0bb8d namespace: default resourceVersion: "6666" uid: 86cd1abb-cff8-4d05-b258-c40634ee8e2d reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-128-181.ec2.internal source: component: kubelet host: ip-10-0-128-181.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:55:19Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-181.ec2.internal uid: e47c1044-c36b-43bc-9e9a-cf4a3752ddb4 kind: Event lastTimestamp: "2026-05-28T14:55:19Z" message: 'Node ip-10-0-128-181.ec2.internal event: Registered Node ip-10-0-128-181.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:55:19Z" name: ip-10-0-128-181.ec2.internal.18b3c27ceaf4ba2c namespace: default resourceVersion: "7978" uid: 8c2a1493-54e5-4eea-829b-af026af9b8fb reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:52:32Z" involvedObject: kind: Node name: ip-10-0-131-234.ec2.internal uid: ip-10-0-131-234.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:52:32Z" message: 'Node ip-10-0-131-234.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:52:32Z" name: ip-10-0-131-234.ec2.internal.18b3c2560b34eec5 namespace: default resourceVersion: "5349" uid: 50c2b1a5-11b3-4c97-a8e0-bf8bde5da905 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:52:32Z" involvedObject: kind: Node name: ip-10-0-131-234.ec2.internal uid: ip-10-0-131-234.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:52:32Z" message: 'Node ip-10-0-131-234.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:52:32Z" name: ip-10-0-131-234.ec2.internal.18b3c2560b3557b9 namespace: default resourceVersion: "5355" uid: 98ee5663-ec8b-4015-9706-efd2befb28fe reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:52:32Z" involvedObject: kind: Node name: ip-10-0-131-234.ec2.internal uid: ip-10-0-131-234.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:52:32Z" message: 'Node ip-10-0-131-234.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:52:32Z" name: ip-10-0-131-234.ec2.internal.18b3c2560b358e06 namespace: default resourceVersion: "5357" uid: 79ff3331-a8ad-48d5-bf7e-93c08677d5f6 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:52:32Z" involvedObject: kind: Node name: ip-10-0-131-234.ec2.internal uid: ip-10-0-131-234.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:52:32Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:52:32Z" name: ip-10-0-131-234.ec2.internal.18b3c2560f08d51f namespace: default resourceVersion: "5305" uid: 65a6b7ab-9ff0-42b5-aeca-05ec82ef9bcb reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:52:32Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-234.ec2.internal resourceVersion: "5307" uid: 4efe272d-5993-4105-81b3-e4404fc08ed9 kind: Event lastTimestamp: "2026-05-28T14:52:32Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T14:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T14:52:32Z" name: ip-10-0-131-234.ec2.internal.18b3c256245983a4 namespace: default resourceVersion: "5392" uid: a2bbf6ae-3bb1-45f5-91f5-104b7c042c56 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:52:36Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-234.ec2.internal uid: 4efe272d-5993-4105-81b3-e4404fc08ed9 kind: Event lastTimestamp: "2026-05-28T14:52:36Z" message: 'Node ip-10-0-131-234.ec2.internal event: Registered Node ip-10-0-131-234.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:52:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:52:36Z" name: ip-10-0-131-234.ec2.internal.18b3c256deeba58c namespace: default resourceVersion: "5488" uid: b2403738-46d3-47d2-91f1-2c3c46eec3f4 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-234.ec2.internal resourceVersion: "5846" uid: 4efe272d-5993-4105-81b3-e4404fc08ed9 kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-131-234.ec2.internal, error getting gateway config for node ip-10-0-131-234.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-131-234.ec2.internal", failed to update chassis to local for local node ip-10-0-131-234.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-131-234.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-131-234.ec2.internal]' metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-131-234 operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-131-234.ec2.internal.18b3c25cadba5ecb namespace: default resourceVersion: "5854" uid: fb79a2d6-a281-4623-bd72-6e2dfc6a31d5 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:07Z" involvedObject: kind: Node name: ip-10-0-131-234.ec2.internal uid: ip-10-0-131-234.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:07Z" message: 'Node ip-10-0-131-234.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T14:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:07Z" name: ip-10-0-131-234.ec2.internal.18b3c25e233a05a8 namespace: default resourceVersion: "6157" uid: 08f6f79e-81a0-4995-bb4e-e12b95f60c25 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:55:19Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-234.ec2.internal uid: 4efe272d-5993-4105-81b3-e4404fc08ed9 kind: Event lastTimestamp: "2026-05-28T14:55:19Z" message: 'Node ip-10-0-131-234.ec2.internal event: Registered Node ip-10-0-131-234.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:55:19Z" name: ip-10-0-131-234.ec2.internal.18b3c27ceaf676c1 namespace: default resourceVersion: "7979" uid: be43515c-a350-4da6-b031-cf10ded2819d reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c83120088 namespace: default resourceVersion: "5727" uid: 198f17fb-a81a-4063-b9de-478624b4a75a reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: 'Node ip-10-0-137-84.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c84e401d1 namespace: default resourceVersion: "5829" uid: 9f4fa034-4e23-4468-9178-9abb95a8cde8 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: 'Node ip-10-0-137-84.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c84e44a66 namespace: default resourceVersion: "5832" uid: 80ae1eb7-724c-475c-ab5c-37ecb030fd5d reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: 'Node ip-10-0-137-84.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c84e471ba namespace: default resourceVersion: "5833" uid: e2214325-d022-4538-a8c3-7547435e401e reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c876f36bf namespace: default resourceVersion: "5732" uid: 29e3c0b7-1c14-46ee-8b55-69769fa572ce reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:00Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-84.ec2.internal resourceVersion: "5733" uid: 6b6c952a-7aa6-4f4b-8c47-6113707ae2f7 kind: Event lastTimestamp: "2026-05-28T14:53:00Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T14:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T14:53:00Z" name: ip-10-0-137-84.ec2.internal.18b3c25c9d106e5a namespace: default resourceVersion: "5839" uid: 13b8b1b8-d69b-4583-bc4b-2ff7b3178a5e reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-84.ec2.internal uid: 6b6c952a-7aa6-4f4b-8c47-6113707ae2f7 kind: Event lastTimestamp: "2026-05-28T14:53:01Z" message: 'Node ip-10-0-137-84.ec2.internal event: Registered Node ip-10-0-137-84.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:53:01Z" name: ip-10-0-137-84.ec2.internal.18b3c25cb13d3784 namespace: default resourceVersion: "5860" uid: 56a48c90-3207-486a-b125-245250c8eee2 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:28Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-84.ec2.internal resourceVersion: "6566" uid: 6b6c952a-7aa6-4f4b-8c47-6113707ae2f7 kind: Event lastTimestamp: "2026-05-28T14:53:28Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-84.ec2.internal, error getting gateway config for node ip-10-0-137-84.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-137-84.ec2.internal", failed to update chassis to local for local node ip-10-0-137-84.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-137-84.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-84.ec2.internal]' metadata: creationTimestamp: "2026-05-28T14:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-137-84 operation: Update time: "2026-05-28T14:53:28Z" name: ip-10-0-137-84.ec2.internal.18b3c26310399ca9 namespace: default resourceVersion: "6574" uid: 31a110bb-d139-460b-92fc-ce83e0cd425f reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:53:35Z" involvedObject: kind: Node name: ip-10-0-137-84.ec2.internal uid: ip-10-0-137-84.ec2.internal kind: Event lastTimestamp: "2026-05-28T14:53:35Z" message: 'Node ip-10-0-137-84.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T14:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T14:53:35Z" name: ip-10-0-137-84.ec2.internal.18b3c264a5408807 namespace: default resourceVersion: "6692" uid: e43f62f8-9a6c-4405-a95d-12fc8c0f313f reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-137-84.ec2.internal source: component: kubelet host: ip-10-0-137-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:55:19Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-84.ec2.internal uid: 6b6c952a-7aa6-4f4b-8c47-6113707ae2f7 kind: Event lastTimestamp: "2026-05-28T14:55:19Z" message: 'Node ip-10-0-137-84.ec2.internal event: Registered Node ip-10-0-137-84.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T14:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T14:55:19Z" name: ip-10-0-137-84.ec2.internal.18b3c27ceaf6965b namespace: default resourceVersion: "7982" uid: 6072837e-a398-4acf-9dab-87cd10a83a72 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T15:01:47Z" involvedObject: apiVersion: v1 kind: Namespace name: kuadrant-system resourceVersion: "16884" uid: 8fcecf56-5fb5-4c43-823e-dabaa2d3b88f kind: Event lastTimestamp: "2026-05-28T15:01:50Z" message: 'error using catalogsource kuadrant-system/kuadrant-operator-catalog: failed to list bundles: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"' metadata: creationTimestamp: "2026-05-28T15:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: catalog operation: Update time: "2026-05-28T15:01:50Z" name: kuadrant-system.18b3c2d74473d506 namespace: default resourceVersion: "17108" uid: fa14e25a-3477-4c01-84ff-2924a42bbaae reason: ResolutionFailed reportingComponent: operator-lifecycle-manager reportingInstance: "" source: component: operator-lifecycle-manager type: Warning - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Received signal to terminate, becoming unready, but keeping serving metadata: creationTimestamp: "2026-05-28T14:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T14:54:56Z" name: kube-system.18b3c27788178847 namespace: default resourceVersion: "7606" uid: 65b88fae-c3b1-468c-abae-50e5702c9a51 reason: TerminationStart reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-5f587d68b9-tkz2d type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: The minimal shutdown duration of 15s finished metadata: creationTimestamp: "2026-05-28T14:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T14:55:11Z" name: kube-system.18b3c27b06904964 namespace: default resourceVersion: "7878" uid: a6279470-b35e-424e-b446-04a338f5a46f reason: TerminationMinimalShutdownDurationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-5f587d68b9-tkz2d type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Server has stopped listening metadata: creationTimestamp: "2026-05-28T14:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T14:55:11Z" name: kube-system.18b3c27b06d25556 namespace: default resourceVersion: "7879" uid: fd297dc2-9cb1-4fc8-ac75-4bf442a40d08 reason: TerminationStoppedServing reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-5f587d68b9-tkz2d type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pre-shutdown hooks have been finished metadata: creationTimestamp: "2026-05-28T14:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T14:55:11Z" name: kube-system.18b3c27b0728b5b1 namespace: default resourceVersion: "7880" uid: 311ced3a-c1d8-4de8-94bb-ee023362821b reason: TerminationPreShutdownHooksFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-5f587d68b9-tkz2d type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pending requests processed metadata: creationTimestamp: "2026-05-28T14:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T14:56:11Z" name: kube-system.18b3c288ff29eccd namespace: default resourceVersion: "10087" uid: 183c9439-effc-4abc-85c5-eb2b2cfa2265 reason: TerminationGracefulTerminationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-5f587d68b9-tkz2d type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T15:03:00Z" involvedObject: apiVersion: v1 kind: Endpoints name: maas-default-gateway-openshift-default kind: Event lastTimestamp: "2026-05-28T15:03:00Z" message: 'Failed to create endpoint for service openshift-ingress/maas-default-gateway-openshift-default: endpoints "maas-default-gateway-openshift-default" already exists' metadata: creationTimestamp: "2026-05-28T15:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T15:03:00Z" name: maas-default-gateway-openshift-default.18b3c2e853319a63 namespace: default resourceVersion: "21623" uid: 258853df-275e-4eb8-9a71-eee518d515d6 reason: FailedToCreateEndpoint reportingComponent: endpoint-controller reportingInstance: "" source: component: endpoint-controller type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T14:47:41Z" involvedObject: apiVersion: v1 kind: Namespace name: openshift-kube-apiserver namespace: default kind: Event lastTimestamp: "2026-05-28T14:47:41Z" message: readyz=true metadata: creationTimestamp: "2026-05-28T14:47:41Z" name: openshift-kube-apiserver.18b3c2124e51c92e namespace: default resourceVersion: "274" uid: 796d3b6d-1ca5-4ffa-ae13-333677d95f47 reason: KubeAPIReadyz reportingComponent: "" reportingInstance: "" source: component: apiserver host: kube-apiserver-795fb547d8-pw4l9 type: Warning kind: EventList metadata: resourceVersion: "47814"