--- apiVersion: v1 items: - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:59Z" involvedObject: kind: CertificateSigningRequest name: csr-24p68 kind: Event lastTimestamp: "2026-05-21T03:19:59Z" message: CSR "csr-24p68" has been approved metadata: creationTimestamp: "2026-05-21T03:19:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:19:59Z" name: csr-24p68.18b1767b47fecbc6 namespace: default resourceVersion: "6209" uid: 49c50236-bbcb-4930-9d9d-16dd57274a39 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:44Z" involvedObject: kind: CertificateSigningRequest name: csr-28pkn kind: Event lastTimestamp: "2026-05-21T03:20:44Z" message: CSR "csr-28pkn" has been approved metadata: creationTimestamp: "2026-05-21T03:20:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:20:44Z" name: csr-28pkn.18b17685d409e14a namespace: default resourceVersion: "7149" uid: 13e1dcac-fa6a-4971-b0aa-eb4d6ef3477b reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:06Z" involvedObject: kind: CertificateSigningRequest name: csr-ck68s kind: Event lastTimestamp: "2026-05-21T03:20:06Z" message: CSR "csr-ck68s" has been approved metadata: creationTimestamp: "2026-05-21T03:20:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:20:06Z" name: csr-ck68s.18b1767cc3f0a5e0 namespace: default resourceVersion: "6320" uid: cd2c17f6-fbba-453e-9dcd-c61c9a09b6e2 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:01Z" involvedObject: kind: CertificateSigningRequest name: csr-fmdms kind: Event lastTimestamp: "2026-05-21T03:20:01Z" message: CSR "csr-fmdms" has been approved metadata: creationTimestamp: "2026-05-21T03:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:20:01Z" name: csr-fmdms.18b1767bd081e45b namespace: default resourceVersion: "6250" uid: d7e89095-c30b-4a2f-8b0e-fa509429f2dd reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:38Z" involvedObject: kind: CertificateSigningRequest name: csr-qjdc7 kind: Event lastTimestamp: "2026-05-21T03:20:38Z" message: CSR "csr-qjdc7" has been approved metadata: creationTimestamp: "2026-05-21T03:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:20:38Z" name: csr-qjdc7.18b1768445618747 namespace: default resourceVersion: "6952" uid: b42c4a5d-4bd7-4faa-ade5-22c52bdb322a reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:08Z" involvedObject: kind: CertificateSigningRequest name: csr-sd8p8 kind: Event lastTimestamp: "2026-05-21T03:20:08Z" message: CSR "csr-sd8p8" has been approved metadata: creationTimestamp: "2026-05-21T03:20:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-21T03:20:08Z" name: csr-sd8p8.18b1767d51989b4f namespace: default resourceVersion: "6372" uid: d2418c65-3834-4181-9e5f-b8f006e39313 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-21T03:24:35Z" involvedObject: apiVersion: datasciencecluster.opendatahub.io/v2 kind: DataScienceCluster name: default-dsc resourceVersion: "13709" uid: 8b19c703-7b05-4dbb-b239-004c2d87bd88 kind: Event lastTimestamp: "2026-05-21T03:24:38Z" message: 'failure deploying resource {map[apiVersion:components.platform.opendatahub.io/v1alpha1 kind:Kserve metadata:map[annotations:map[component.opendatahub.io/management-state:Managed platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-dsc platform.opendatahub.io/instance.uid:8b19c703-7b05-4dbb-b239-004c2d87bd88 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1] creationTimestamp: labels:map[platform.opendatahub.io/part-of:datasciencecluster] name:default-kserve ownerReferences:[map[apiVersion:datasciencecluster.opendatahub.io/v2 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:DataScienceCluster name:default-dsc uid:8b19c703-7b05-4dbb-b239-004c2d87bd88]]] spec:map[modelsAsService:map[managementState:Removed] nim:map[managementState:Managed] rawDeploymentServiceConfig:Headed] status:map[]]}: apply failed components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve default-kserve: kserves.components.platform.opendatahub.io "default-kserve" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion datasciencecluster.opendatahub.io/v2 Kind DataScienceCluster: no matches for kind "DataScienceCluster" in version "datasciencecluster.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-21T03:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-21T03:24:38Z" name: default-dsc.18b176bb791e6e70 namespace: default resourceVersion: "13770" uid: 6929f2b7-3a98-43c6-b38d-dfc57f03fdc5 reason: ProvisioningError reportingComponent: datasciencecluster reportingInstance: "" source: component: datasciencecluster type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-21T03:24:35Z" involvedObject: apiVersion: dscinitialization.opendatahub.io/v2 kind: DSCInitialization name: default-dsci resourceVersion: "13572" uid: fbe7e19a-ac69-40e1-bed6-dc32f8c96a5d kind: Event lastTimestamp: "2026-05-21T03:24:37Z" message: 'failed to create operator resources for instance default-dsci: unable to patch networking.k8s.io/v1, Kind=NetworkPolicy opendatahub/opendatahub: networkpolicies.networking.k8s.io "opendatahub" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion dscinitialization.opendatahub.io/v2 Kind DSCInitialization: no matches for kind "DSCInitialization" in version "dscinitialization.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-21T03:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-21T03:24:37Z" name: default-dsci.18b176bb6715c5b4 namespace: default resourceVersion: "13768" uid: eeab59b8-c5b7-4dc1-91d5-fceb3da5e6e1 reason: DSCInitializationReconcileError reportingComponent: dscinitialization-controller reportingInstance: "" source: component: dscinitialization-controller type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-21T03:24:42Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "13898" uid: bf0cfccb-da0a-486d-ac21-62453cd76404 kind: Event lastTimestamp: "2026-05-21T03:24:50Z" message: 'failed to lookup object openshift-ingress/data-science-tls-rule: no matches for kind "DestinationRule" in version "networking.istio.io/v1"' metadata: creationTimestamp: "2026-05-21T03:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-21T03:24:50Z" name: default-gateway.18b176bd19e55cff namespace: default resourceVersion: "14111" uid: d3cb032c-7b0f-4cc4-b177-6d918b7b6df8 reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-21T03:24:51Z" involvedObject: apiVersion: components.platform.opendatahub.io/v1alpha1 kind: Kserve name: default-kserve resourceVersion: "14197" uid: 953779f9-2610-4505-b56d-3d1421810ca3 kind: Event lastTimestamp: "2026-05-21T03:25:17Z" message: "failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:953779f9-2610-4505-b56d-3d1421810ca3 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:953779f9-2610-4505-b56d-3d1421810ca3]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n \ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n \ echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n \ echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n \ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n \ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n \ if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n \ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n \ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n \ # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n \ gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n \ fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n \ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n \ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n \ fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n \ echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n \ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n \ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n \ export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n \ echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"" metadata: creationTimestamp: "2026-05-21T03:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-21T03:25:17Z" name: default-kserve.18b176bf205067e7 namespace: default resourceVersion: "15624" uid: 5cd5b802-b336-4041-a7bc-fca379b43d07 reason: ProvisioningError reportingComponent: kserve reportingInstance: "" source: component: kserve type: Warning - apiVersion: v1 count: 32 eventTime: null firstTimestamp: "2026-05-21T03:24:40Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Monitoring name: default-monitoring resourceVersion: "13788" uid: 70e5cbbb-d43e-4b60-bec4-6a608aea1e73 kind: Event lastTimestamp: "2026-05-21T03:41:48Z" message: 'error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache' metadata: creationTimestamp: "2026-05-21T03:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-21T03:41:48Z" name: default-monitoring.18b176bcace63077 namespace: default resourceVersion: "35061" uid: 064963a4-b69c-4156-b595-8b38d361414e reason: ProvisioningError reportingComponent: monitoring reportingInstance: "" source: component: monitoring type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:32Z" involvedObject: kind: Node name: ip-10-0-128-161.ec2.internal uid: ip-10-0-128-161.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:32Z" message: 'Node ip-10-0-128-161.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-21T03:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:32Z" name: ip-10-0-128-161.ec2.internal.18b17674f483d541 namespace: default resourceVersion: "5549" uid: 21ab39a0-f0b4-4f35-9e7a-a5f0bd5f0552 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-128-161.ec2.internal source: component: kubelet host: ip-10-0-128-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:32Z" involvedObject: kind: Node name: ip-10-0-128-161.ec2.internal uid: ip-10-0-128-161.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:32Z" message: 'Node ip-10-0-128-161.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-21T03:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:32Z" name: ip-10-0-128-161.ec2.internal.18b17674f48419a9 namespace: default resourceVersion: "5556" uid: 98e93e6c-5e37-468b-90ef-0048c4672d2a reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-128-161.ec2.internal source: component: kubelet host: ip-10-0-128-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:32Z" involvedObject: kind: Node name: ip-10-0-128-161.ec2.internal uid: ip-10-0-128-161.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:32Z" message: 'Node ip-10-0-128-161.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-21T03:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:32Z" name: ip-10-0-128-161.ec2.internal.18b17674f4843ede namespace: default resourceVersion: "5558" uid: bb9cd3b0-f09a-4a4c-8932-260e0e33c5df reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-128-161.ec2.internal source: component: kubelet host: ip-10-0-128-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:32Z" involvedObject: kind: Node name: ip-10-0-128-161.ec2.internal uid: ip-10-0-128-161.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:32Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-21T03:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:32Z" name: ip-10-0-128-161.ec2.internal.18b17674f761ccea namespace: default resourceVersion: "5506" uid: 837ea092-a8e6-4fb3-b61f-09d3f3fe4038 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-128-161.ec2.internal source: component: kubelet host: ip-10-0-128-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:32Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-161.ec2.internal resourceVersion: "5508" uid: ddfc8cd7-98b3-4207-8df0-b38ba590b588 kind: Event lastTimestamp: "2026-05-21T03:19:32Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-21T03:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-21T03:19:32Z" name: ip-10-0-128-161.ec2.internal.18b176750dee4a32 namespace: default resourceVersion: "5579" uid: 98c01020-092d-4626-a9f2-582eded5d63e reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:35Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-161.ec2.internal uid: ddfc8cd7-98b3-4207-8df0-b38ba590b588 kind: Event lastTimestamp: "2026-05-21T03:19:35Z" message: 'Node ip-10-0-128-161.ec2.internal event: Registered Node ip-10-0-128-161.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:19:35Z" name: ip-10-0-128-161.ec2.internal.18b17675b22c7ddc namespace: default resourceVersion: "5681" uid: eecedc8f-f055-4e83-bc08-804c140f4fdc reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:07Z" involvedObject: kind: Node name: ip-10-0-128-161.ec2.internal uid: ip-10-0-128-161.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:07Z" message: 'Node ip-10-0-128-161.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-21T03:20:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:07Z" name: ip-10-0-128-161.ec2.internal.18b1767d0d961a27 namespace: default resourceVersion: "6338" uid: ba8ce3d4-2ed3-42f5-8e54-28d0f5e89a38 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-128-161.ec2.internal source: component: kubelet host: ip-10-0-128-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:21:14Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-128-161.ec2.internal uid: ddfc8cd7-98b3-4207-8df0-b38ba590b588 kind: Event lastTimestamp: "2026-05-21T03:21:14Z" message: 'Node ip-10-0-128-161.ec2.internal event: Registered Node ip-10-0-128-161.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:21:14Z" name: ip-10-0-128-161.ec2.internal.18b1768c9e5d695e namespace: default resourceVersion: "7852" uid: e5ce06f9-ab90-48a8-bed6-0ae006f1446f reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767dfc2799d2 namespace: default resourceVersion: "6431" uid: fc86f917-a19f-406d-92dd-3b6a7a7e0fc6 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: 'Node ip-10-0-131-76.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767dfde4ef66 namespace: default resourceVersion: "6439" uid: c828d565-3d97-4acf-b82e-2d322c59c482 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: 'Node ip-10-0-131-76.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767dfde53431 namespace: default resourceVersion: "6441" uid: 95c56e6a-c84b-4113-abd5-24ed70221c74 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: 'Node ip-10-0-131-76.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767dfde55734 namespace: default resourceVersion: "6443" uid: e85db133-a3f0-40c7-9f99-3f6b2310d685 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767e005545dc namespace: default resourceVersion: "6436" uid: 69b4c675-6258-42b9-b22a-718515cfdccd reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:11Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-76.ec2.internal resourceVersion: "6440" uid: 6f0fe71b-ca2e-49b6-95c0-19ba15c61a78 kind: Event lastTimestamp: "2026-05-21T03:20:11Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-21T03:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-21T03:20:11Z" name: ip-10-0-131-76.ec2.internal.18b1767e16525b74 namespace: default resourceVersion: "6508" uid: 38cf7725-65da-4a86-9660-631d261a5a9c reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:15Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-76.ec2.internal uid: 6f0fe71b-ca2e-49b6-95c0-19ba15c61a78 kind: Event lastTimestamp: "2026-05-21T03:20:15Z" message: 'Node ip-10-0-131-76.ec2.internal event: Registered Node ip-10-0-131-76.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:20:15Z" name: ip-10-0-131-76.ec2.internal.18b1767f02d03c05 namespace: default resourceVersion: "6627" uid: 779a2cda-0041-4c35-a1c2-98917de17b55 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:39Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-76.ec2.internal resourceVersion: "6973" uid: 6f0fe71b-ca2e-49b6-95c0-19ba15c61a78 kind: Event lastTimestamp: "2026-05-21T03:20:39Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-131-76.ec2.internal, error getting gateway config for node ip-10-0-131-76.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-131-76.ec2.internal", failed to update chassis to local for local node ip-10-0-131-76.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-131-76.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-131-76.ec2.internal]' metadata: creationTimestamp: "2026-05-21T03:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-131-76 operation: Update time: "2026-05-21T03:20:39Z" name: ip-10-0-131-76.ec2.internal.18b17684a3b037e5 namespace: default resourceVersion: "6975" uid: 2e41ce59-5593-4da5-8b15-290e096fadbc reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:46Z" involvedObject: kind: Node name: ip-10-0-131-76.ec2.internal uid: ip-10-0-131-76.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:46Z" message: 'Node ip-10-0-131-76.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-21T03:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:46Z" name: ip-10-0-131-76.ec2.internal.18b1768636bebf79 namespace: default resourceVersion: "7178" uid: cf2d20ce-7c29-43c0-af40-41cf1e373f3f reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-131-76.ec2.internal source: component: kubelet host: ip-10-0-131-76.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:21:14Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-131-76.ec2.internal uid: 6f0fe71b-ca2e-49b6-95c0-19ba15c61a78 kind: Event lastTimestamp: "2026-05-21T03:21:14Z" message: 'Node ip-10-0-131-76.ec2.internal event: Registered Node ip-10-0-131-76.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:21:14Z" name: ip-10-0-131-76.ec2.internal.18b1768c9e5e7ec2 namespace: default resourceVersion: "7856" uid: 238cc105-a254-43a0-8f19-2c252027bb16 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675cad44c54 namespace: default resourceVersion: "5687" uid: f0b94ca2-f52e-4778-a7d2-d639873435c0 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: 'Node ip-10-0-137-251.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675cc7907dc namespace: default resourceVersion: "5775" uid: 4830401b-cc2d-4a90-982a-9dd5681cea91 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: 'Node ip-10-0-137-251.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675cc79486c namespace: default resourceVersion: "5776" uid: 1a638eae-4013-4d26-ab29-d04c823ffb20 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: 'Node ip-10-0-137-251.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675cc7970a2 namespace: default resourceVersion: "5777" uid: 5286c586-e03a-4798-bf63-57b50527f9ec reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675cf43d1bf namespace: default resourceVersion: "5692" uid: 49dfdb80-4e7c-4447-8601-ae6bcaad750c reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:36Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-251.ec2.internal resourceVersion: "5693" uid: 78f7e656-f459-46ba-9235-dfbb92f8e4d9 kind: Event lastTimestamp: "2026-05-21T03:19:36Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-21T03:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-21T03:19:36Z" name: ip-10-0-137-251.ec2.internal.18b17675e3d4154b namespace: default resourceVersion: "5784" uid: 5ee57ce4-5c89-4430-ba72-152b65a9ed85 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:19:40Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-251.ec2.internal uid: 78f7e656-f459-46ba-9235-dfbb92f8e4d9 kind: Event lastTimestamp: "2026-05-21T03:19:40Z" message: 'Node ip-10-0-137-251.ec2.internal event: Registered Node ip-10-0-137-251.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:19:40Z" name: ip-10-0-137-251.ec2.internal.18b17676dc44a1f4 namespace: default resourceVersion: "5870" uid: 21e08100-e5c1-400b-ba88-be32c45a685d reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:03Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-251.ec2.internal resourceVersion: "6282" uid: 78f7e656-f459-46ba-9235-dfbb92f8e4d9 kind: Event lastTimestamp: "2026-05-21T03:20:03Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-251.ec2.internal, error getting gateway config for node ip-10-0-137-251.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-137-251.ec2.internal", failed to update chassis to local for local node ip-10-0-137-251.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-137-251.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-251.ec2.internal]' metadata: creationTimestamp: "2026-05-21T03:20:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-137-251 operation: Update time: "2026-05-21T03:20:03Z" name: ip-10-0-137-251.ec2.internal.18b1767c2d0897db namespace: default resourceVersion: "6284" uid: df48574f-1261-408c-9ea4-0073e51cc688 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:20:09Z" involvedObject: kind: Node name: ip-10-0-137-251.ec2.internal uid: ip-10-0-137-251.ec2.internal kind: Event lastTimestamp: "2026-05-21T03:20:09Z" message: 'Node ip-10-0-137-251.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-21T03:20:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-21T03:20:09Z" name: ip-10-0-137-251.ec2.internal.18b1767db0197278 namespace: default resourceVersion: "6394" uid: 4d9c944f-c206-4573-9530-f0dd4b5ba0f2 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-137-251.ec2.internal source: component: kubelet host: ip-10-0-137-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:21:14Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-251.ec2.internal uid: 78f7e656-f459-46ba-9235-dfbb92f8e4d9 kind: Event lastTimestamp: "2026-05-21T03:21:14Z" message: 'Node ip-10-0-137-251.ec2.internal event: Registered Node ip-10-0-137-251.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-21T03:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-21T03:21:14Z" name: ip-10-0-137-251.ec2.internal.18b1768c9e5e992a namespace: default resourceVersion: "7859" uid: b7eca5de-9c1c-4ae6-b967-e0f1a0f927fd reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-21T03:25:47Z" involvedObject: apiVersion: v1 kind: Namespace name: kuadrant-system resourceVersion: "15890" uid: f04d697b-a1f0-4692-a26e-16642ac8e9c3 kind: Event lastTimestamp: "2026-05-21T03:25:50Z" message: 'error using catalogsource kuadrant-system/kuadrant-operator-catalog: failed to list bundles: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"' metadata: creationTimestamp: "2026-05-21T03:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: catalog operation: Update time: "2026-05-21T03:25:51Z" name: kuadrant-system.18b176cc4108639c namespace: default resourceVersion: "16104" uid: 7ee02b45-7f6a-4a4d-9973-555a3cd2afde reason: ResolutionFailed reportingComponent: operator-lifecycle-manager reportingInstance: "" source: component: operator-lifecycle-manager type: Warning - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Received signal to terminate, becoming unready, but keeping serving metadata: creationTimestamp: "2026-05-21T03:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-21T03:20:53Z" name: kube-system.18b17687da12b428 namespace: default resourceVersion: "7494" uid: 818fdcf9-89d9-41b4-9bb8-f603e7e1f1c9 reason: TerminationStart reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-54c5fd76f9-xbdwr type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: The minimal shutdown duration of 15s finished metadata: creationTimestamp: "2026-05-21T03:21:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-21T03:21:08Z" name: kube-system.18b1768b58afebe7 namespace: default resourceVersion: "7778" uid: ffb64846-6908-4752-bada-698b2e819d7d reason: TerminationMinimalShutdownDurationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-54c5fd76f9-xbdwr type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Server has stopped listening metadata: creationTimestamp: "2026-05-21T03:21:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-21T03:21:08Z" name: kube-system.18b1768b591f326d namespace: default resourceVersion: "7779" uid: 148e8d8d-1efe-40c0-be8e-cd5275e753a4 reason: TerminationStoppedServing reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-54c5fd76f9-xbdwr type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pre-shutdown hooks have been finished metadata: creationTimestamp: "2026-05-21T03:21:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-21T03:21:08Z" name: kube-system.18b1768b596bd110 namespace: default resourceVersion: "7780" uid: 9dd827a7-d339-4057-8647-28be76069497 reason: TerminationPreShutdownHooksFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-54c5fd76f9-xbdwr type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pending requests processed metadata: creationTimestamp: "2026-05-21T03:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-21T03:22:08Z" name: kube-system.18b176995176f187 namespace: default resourceVersion: "10152" uid: 79943a1c-e935-4c56-9525-297e8f115a4e reason: TerminationGracefulTerminationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-54c5fd76f9-xbdwr type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-21T03:14:11Z" involvedObject: apiVersion: v1 kind: Namespace name: openshift-kube-apiserver namespace: default kind: Event lastTimestamp: "2026-05-21T03:14:11Z" message: readyz=true metadata: creationTimestamp: "2026-05-21T03:14:11Z" name: openshift-kube-apiserver.18b1762a3bc67220 namespace: default resourceVersion: "274" uid: 86e5a432-1087-48b8-a6f0-ef821b4a6867 reason: KubeAPIReadyz reportingComponent: "" reportingInstance: "" source: component: apiserver host: kube-apiserver-78fccb9db-mq6r8 type: Warning kind: EventList metadata: resourceVersion: "46625"