--- apiVersion: v1 items: - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:57Z" involvedObject: kind: CertificateSigningRequest name: csr-4s59q kind: Event lastTimestamp: "2026-05-26T19:47:57Z" message: CSR "csr-4s59q" has been approved metadata: creationTimestamp: "2026-05-26T19:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:47:57Z" name: csr-4s59q.18b3354bbae6e34e namespace: default resourceVersion: "6541" uid: e17e15d6-5ca9-480c-8896-b85dd061ec0e reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:48:21Z" involvedObject: kind: CertificateSigningRequest name: csr-5cwlg kind: Event lastTimestamp: "2026-05-26T19:48:21Z" message: CSR "csr-5cwlg" has been approved metadata: creationTimestamp: "2026-05-26T19:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:48:21Z" name: csr-5cwlg.18b3355163217ac6 namespace: default resourceVersion: "6933" uid: 2979e4b9-654c-4935-b05f-e27c571b428d reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:48:15Z" involvedObject: kind: CertificateSigningRequest name: csr-8nz6q kind: Event lastTimestamp: "2026-05-26T19:48:15Z" message: CSR "csr-8nz6q" has been approved metadata: creationTimestamp: "2026-05-26T19:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:48:15Z" name: csr-8nz6q.18b3355014243514 namespace: default resourceVersion: "6850" uid: 51b8e65d-f064-48db-9163-bf4f88efe48b reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:44Z" involvedObject: kind: CertificateSigningRequest name: csr-hgdpw kind: Event lastTimestamp: "2026-05-26T19:47:44Z" message: CSR "csr-hgdpw" has been approved metadata: creationTimestamp: "2026-05-26T19:47:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:47:44Z" name: csr-hgdpw.18b33548b230e450 namespace: default resourceVersion: "6058" uid: 286e8ec8-2d23-4fe8-b62d-d2d5e082af74 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:51Z" involvedObject: kind: CertificateSigningRequest name: csr-ptwtb kind: Event lastTimestamp: "2026-05-26T19:47:51Z" message: CSR "csr-ptwtb" has been approved metadata: creationTimestamp: "2026-05-26T19:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:47:51Z" name: csr-ptwtb.18b3354a5b33ec2f namespace: default resourceVersion: "6356" uid: 03551df1-d245-4262-9724-8c013dc4ae9c reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:50Z" involvedObject: kind: CertificateSigningRequest name: csr-xbb7m kind: Event lastTimestamp: "2026-05-26T19:47:50Z" message: CSR "csr-xbb7m" has been approved metadata: creationTimestamp: "2026-05-26T19:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-26T19:47:50Z" name: csr-xbb7m.18b3354a19aa27f6 namespace: default resourceVersion: "6329" uid: b8f00433-5268-46e6-88f4-90b336976854 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T19:57:39Z" involvedObject: apiVersion: datasciencecluster.opendatahub.io/v2 kind: DataScienceCluster name: default-dsc resourceVersion: "15132" uid: c9efe38b-c4d5-43c7-a7bc-5cd03cac39a8 kind: Event lastTimestamp: "2026-05-26T19:57:40Z" message: 'failure deploying resource {map[apiVersion:components.platform.opendatahub.io/v1alpha1 kind:Kserve metadata:map[annotations:map[component.opendatahub.io/management-state:Managed platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-dsc platform.opendatahub.io/instance.uid:c9efe38b-c4d5-43c7-a7bc-5cd03cac39a8 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1] creationTimestamp: labels:map[platform.opendatahub.io/part-of:datasciencecluster] name:default-kserve ownerReferences:[map[apiVersion:datasciencecluster.opendatahub.io/v2 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:DataScienceCluster name:default-dsc uid:c9efe38b-c4d5-43c7-a7bc-5cd03cac39a8]]] spec:map[modelsAsService:map[managementState:Removed] nim:map[managementState:Managed] rawDeploymentServiceConfig:Headed] status:map[]]}: apply failed components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve default-kserve: kserves.components.platform.opendatahub.io "default-kserve" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion datasciencecluster.opendatahub.io/v2 Kind DataScienceCluster: no matches for kind "DataScienceCluster" in version "datasciencecluster.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-26T19:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T19:57:40Z" name: default-dsc.18b335d343f11bb7 namespace: default resourceVersion: "15192" uid: 0ded2d51-9755-41ff-bd6e-b6915c83a578 reason: ProvisioningError reportingComponent: datasciencecluster reportingInstance: "" source: component: datasciencecluster type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T19:57:38Z" involvedObject: apiVersion: dscinitialization.opendatahub.io/v2 kind: DSCInitialization name: default-dsci resourceVersion: "14965" uid: a19bcc47-17bf-45a5-8a9c-5d49198b94d4 kind: Event lastTimestamp: "2026-05-26T19:57:40Z" message: 'failed to create operator resources for instance default-dsci: unable to patch networking.k8s.io/v1, Kind=NetworkPolicy opendatahub/opendatahub: networkpolicies.networking.k8s.io "opendatahub" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion dscinitialization.opendatahub.io/v2 Kind DSCInitialization: no matches for kind "DSCInitialization" in version "dscinitialization.opendatahub.io/v2"' metadata: creationTimestamp: "2026-05-26T19:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T19:57:40Z" name: default-dsci.18b335d333ad69d7 namespace: default resourceVersion: "15188" uid: 9a4e471a-db6f-4b03-bba7-f2f73cc04e9a reason: DSCInitializationReconcileError reportingComponent: dscinitialization-controller reportingInstance: "" source: component: dscinitialization-controller type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-26T19:57:43Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "15305" uid: a9726e14-2ed6-43a0-9bff-645741ce6edc kind: Event lastTimestamp: "2026-05-26T19:57:50Z" message: 'failed to lookup object openshift-ingress/data-science-tls-rule: no matches for kind "DestinationRule" in version "networking.istio.io/v1"' metadata: creationTimestamp: "2026-05-26T19:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T19:57:50Z" name: default-gateway.18b335d432c08dff namespace: default resourceVersion: "15524" uid: ca2cda85-ad5d-46b4-96c7-6dd952ab966b reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-26T19:57:52Z" involvedObject: apiVersion: components.platform.opendatahub.io/v1alpha1 kind: Kserve name: default-kserve resourceVersion: "15632" uid: f8f07d8e-1413-4881-9e47-584682db1bb7 kind: Event lastTimestamp: "2026-05-26T19:58:20Z" message: "failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:f8f07d8e-1413-4881-9e47-584682db1bb7 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:f8f07d8e-1413-4881-9e47-584682db1bb7]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n \ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n \ echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n \ echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n \ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n \ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n \ if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n \ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n \ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n \ # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n \ gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n \ fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n \ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n \ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n \ fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n \ echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n \ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n \ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n \ export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n \ echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"" metadata: creationTimestamp: "2026-05-26T19:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T19:58:20Z" name: default-kserve.18b335d673c08fb4 namespace: default resourceVersion: "17073" uid: 0601e782-2b5c-47d4-90d7-e35d3f8f1e46 reason: ProvisioningError reportingComponent: kserve reportingInstance: "" source: component: kserve type: Warning - apiVersion: v1 count: 34 eventTime: null firstTimestamp: "2026-05-26T19:57:41Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Monitoring name: default-monitoring resourceVersion: "15203" uid: 620f19b1-4309-45f1-b3a0-ca5bf9dc05ce kind: Event lastTimestamp: "2026-05-26T20:14:51Z" message: 'error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache' metadata: creationTimestamp: "2026-05-26T19:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T20:14:51Z" name: default-monitoring.18b335d3dc59e035 namespace: default resourceVersion: "36285" uid: a014413e-7eec-488f-9c14-7d8a2947bc4c reason: ProvisioningError reportingComponent: monitoring reportingInstance: "" source: component: monitoring type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: kind: Node name: ip-10-0-130-165.ec2.internal uid: ip-10-0-130-165.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: 'Node ip-10-0-130-165.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549e667e4db namespace: default resourceVersion: "6249" uid: 986b9d26-7cb9-4b07-971e-10f853000a72 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-130-165.ec2.internal source: component: kubelet host: ip-10-0-130-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: kind: Node name: ip-10-0-130-165.ec2.internal uid: ip-10-0-130-165.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: 'Node ip-10-0-130-165.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549e66829d5 namespace: default resourceVersion: "6250" uid: 7b477d5b-8fbc-4102-8816-550ef9742b7c reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-130-165.ec2.internal source: component: kubelet host: ip-10-0-130-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: kind: Node name: ip-10-0-130-165.ec2.internal uid: ip-10-0-130-165.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: 'Node ip-10-0-130-165.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549e6684df9 namespace: default resourceVersion: "6251" uid: aa3bf8d7-7ab4-48c2-8a9a-c6760cebcf1e reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-130-165.ec2.internal source: component: kubelet host: ip-10-0-130-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: kind: Node name: ip-10-0-130-165.ec2.internal uid: ip-10-0-130-165.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549e95c1bd3 namespace: default resourceVersion: "6213" uid: 6f62dc53-bebc-4747-bd81-c0c42e39c65e reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-130-165.ec2.internal source: component: kubelet host: ip-10-0-130-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-130-165.ec2.internal uid: ea870359-d734-43e6-818d-2eb57464d9a5 kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: 'Node ip-10-0-130-165.ec2.internal event: Registered Node ip-10-0-130-165.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549fcb38481 namespace: default resourceVersion: "6278" uid: 1f92b3f0-7e21-41ae-8d24-2ad2b19ca311 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:49Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-130-165.ec2.internal resourceVersion: "6218" uid: ea870359-d734-43e6-818d-2eb57464d9a5 kind: Event lastTimestamp: "2026-05-26T19:47:49Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-26T19:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-26T19:47:49Z" name: ip-10-0-130-165.ec2.internal.18b33549fec2db77 namespace: default resourceVersion: "6285" uid: 447104b1-3249-40b4-b2fc-fe00ca91b68a reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:48:23Z" involvedObject: kind: Node name: ip-10-0-130-165.ec2.internal uid: ip-10-0-130-165.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:48:23Z" message: 'Node ip-10-0-130-165.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-26T19:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:48:23Z" name: ip-10-0-130-165.ec2.internal.18b33551c32422bd namespace: default resourceVersion: "6942" uid: 3ba01e44-1924-4399-a6cd-6d7a9b2ce87d reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-130-165.ec2.internal source: component: kubelet host: ip-10-0-130-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:50:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-130-165.ec2.internal uid: ea870359-d734-43e6-818d-2eb57464d9a5 kind: Event lastTimestamp: "2026-05-26T19:50:01Z" message: 'Node ip-10-0-130-165.ec2.internal event: Registered Node ip-10-0-130-165.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:50:01Z" name: ip-10-0-130-165.ec2.internal.18b33568aff640df namespace: default resourceVersion: "8056" uid: 9851fedc-084e-423e-9983-ddd087c48672 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b335441dd66f6d namespace: default resourceVersion: "5648" uid: 9e90a4f5-c72c-462a-b517-98e9d3caf046 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: 'Node ip-10-0-136-217.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b335441fb18c2a namespace: default resourceVersion: "5719" uid: 54d106ea-2a17-4320-9317-12576c9ee554 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: 'Node ip-10-0-136-217.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b335441fb1d5ec namespace: default resourceVersion: "5726" uid: c2c3421b-186f-4202-9abd-6c4b626dc6ef reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: 'Node ip-10-0-136-217.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b335441fb1fb09 namespace: default resourceVersion: "5737" uid: 56d8424f-4328-4cee-9778-73b5ecb222af reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b33544229dcf26 namespace: default resourceVersion: "5653" uid: 4ffa05d6-704b-4c19-b441-ecb1f05d1fb6 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-136-217.ec2.internal uid: 79cf29f9-acfe-425e-9955-0906981f1e3f kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: 'Node ip-10-0-136-217.ec2.internal event: Registered Node ip-10-0-136-217.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b335442a672347 namespace: default resourceVersion: "5673" uid: fb4b0522-4ff3-4a62-bc26-d009702c682e reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:24Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-136-217.ec2.internal resourceVersion: "5655" uid: 79cf29f9-acfe-425e-9955-0906981f1e3f kind: Event lastTimestamp: "2026-05-26T19:47:24Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-26T19:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-26T19:47:24Z" name: ip-10-0-136-217.ec2.internal.18b33544394cf294 namespace: default resourceVersion: "5748" uid: 80882576-3783-482b-8ce3-1a0b716d2129 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:52Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-136-217.ec2.internal resourceVersion: "6468" uid: 79cf29f9-acfe-425e-9955-0906981f1e3f kind: Event lastTimestamp: "2026-05-26T19:47:52Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-136-217.ec2.internal, error getting gateway config for node ip-10-0-136-217.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-136-217.ec2.internal", failed to update chassis to local for local node ip-10-0-136-217.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-136-217.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-136-217.ec2.internal]' metadata: creationTimestamp: "2026-05-26T19:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-136-217 operation: Update time: "2026-05-26T19:47:52Z" name: ip-10-0-136-217.ec2.internal.18b3354ab8499a66 namespace: default resourceVersion: "6471" uid: be900c62-5953-43fc-b765-ab3412d7a4ce reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:58Z" involvedObject: kind: Node name: ip-10-0-136-217.ec2.internal uid: ip-10-0-136-217.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:58Z" message: 'Node ip-10-0-136-217.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-26T19:47:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:58Z" name: ip-10-0-136-217.ec2.internal.18b3354c19177a57 namespace: default resourceVersion: "6563" uid: ad6e9139-1b26-43f2-93fa-aa7fcb247064 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-136-217.ec2.internal source: component: kubelet host: ip-10-0-136-217.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:50:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-136-217.ec2.internal uid: 79cf29f9-acfe-425e-9955-0906981f1e3f kind: Event lastTimestamp: "2026-05-26T19:50:01Z" message: 'Node ip-10-0-136-217.ec2.internal event: Registered Node ip-10-0-136-217.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:50:01Z" name: ip-10-0-136-217.ec2.internal.18b33568aff6cfce namespace: default resourceVersion: "8059" uid: e48095b5-fe8e-4a2b-ab25-8c0ef0495160 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:17Z" involvedObject: kind: Node name: ip-10-0-141-124.ec2.internal uid: ip-10-0-141-124.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:17Z" message: 'Node ip-10-0-141-124.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-26T19:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:17Z" name: ip-10-0-141-124.ec2.internal.18b335427396bc0a namespace: default resourceVersion: "5465" uid: c3be5ac8-751b-43df-b7c0-3a93c5f63607 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-141-124.ec2.internal source: component: kubelet host: ip-10-0-141-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:17Z" involvedObject: kind: Node name: ip-10-0-141-124.ec2.internal uid: ip-10-0-141-124.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:17Z" message: 'Node ip-10-0-141-124.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-26T19:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:17Z" name: ip-10-0-141-124.ec2.internal.18b33542739715f4 namespace: default resourceVersion: "5466" uid: d759dd1b-f265-47ee-b3d4-5ffa9eb68cd1 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-141-124.ec2.internal source: component: kubelet host: ip-10-0-141-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T19:47:17Z" involvedObject: kind: Node name: ip-10-0-141-124.ec2.internal uid: ip-10-0-141-124.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:17Z" message: 'Node ip-10-0-141-124.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-26T19:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:17Z" name: ip-10-0-141-124.ec2.internal.18b335427397387f namespace: default resourceVersion: "5473" uid: cbb4651d-03bd-4429-af15-5a71e8aea123 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-141-124.ec2.internal source: component: kubelet host: ip-10-0-141-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:17Z" involvedObject: kind: Node name: ip-10-0-141-124.ec2.internal uid: ip-10-0-141-124.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:17Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-26T19:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:17Z" name: ip-10-0-141-124.ec2.internal.18b3354276cd2b13 namespace: default resourceVersion: "5420" uid: 255c143f-073f-404a-8268-d660d0b51f50 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-141-124.ec2.internal source: component: kubelet host: ip-10-0-141-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:17Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-141-124.ec2.internal resourceVersion: "5423" uid: 0d7b8f9c-ca8c-4f70-bb73-153773546daa kind: Event lastTimestamp: "2026-05-26T19:47:17Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-26T19:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-26T19:47:17Z" name: ip-10-0-141-124.ec2.internal.18b335428fe2e40a namespace: default resourceVersion: "5509" uid: 637b0023-aaf3-4a0c-9a22-bccb9ac0f4f3 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:19Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-141-124.ec2.internal uid: 0d7b8f9c-ca8c-4f70-bb73-153773546daa kind: Event lastTimestamp: "2026-05-26T19:47:19Z" message: 'Node ip-10-0-141-124.ec2.internal event: Registered Node ip-10-0-141-124.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:47:19Z" name: ip-10-0-141-124.ec2.internal.18b335430059a8c7 namespace: default resourceVersion: "5576" uid: 248314eb-ab52-4690-b4d9-a5f9a25e2c39 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:45Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-141-124.ec2.internal resourceVersion: "6081" uid: 0d7b8f9c-ca8c-4f70-bb73-153773546daa kind: Event lastTimestamp: "2026-05-26T19:47:45Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-124.ec2.internal, error getting gateway config for node ip-10-0-141-124.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-141-124.ec2.internal", failed to update chassis to local for local node ip-10-0-141-124.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-124.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-124.ec2.internal]' metadata: creationTimestamp: "2026-05-26T19:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-141-124 operation: Update time: "2026-05-26T19:47:45Z" name: ip-10-0-141-124.ec2.internal.18b335490ee472c7 namespace: default resourceVersion: "6084" uid: 0cd416a8-24f5-4416-953e-73c68018bade reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:47:51Z" involvedObject: kind: Node name: ip-10-0-141-124.ec2.internal uid: ip-10-0-141-124.ec2.internal kind: Event lastTimestamp: "2026-05-26T19:47:51Z" message: 'Node ip-10-0-141-124.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-26T19:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T19:47:51Z" name: ip-10-0-141-124.ec2.internal.18b3354a6cebcfd2 namespace: default resourceVersion: "6389" uid: 0cdf529f-df32-4208-9b20-c1fa7ace920d reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-141-124.ec2.internal source: component: kubelet host: ip-10-0-141-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:50:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-141-124.ec2.internal uid: 0d7b8f9c-ca8c-4f70-bb73-153773546daa kind: Event lastTimestamp: "2026-05-26T19:50:01Z" message: 'Node ip-10-0-141-124.ec2.internal event: Registered Node ip-10-0-141-124.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-26T19:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T19:50:01Z" name: ip-10-0-141-124.ec2.internal.18b33568aff6e6c6 namespace: default resourceVersion: "8064" uid: b5a79923-742a-4322-93d1-ab812e784725 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T19:58:51Z" involvedObject: apiVersion: v1 kind: Namespace name: kuadrant-system resourceVersion: "17324" uid: ab880c97-d8a6-47fd-9cf4-e0df708b89f8 kind: Event lastTimestamp: "2026-05-26T19:58:54Z" message: 'error using catalogsource kuadrant-system/kuadrant-operator-catalog: failed to list bundles: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"' metadata: creationTimestamp: "2026-05-26T19:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: catalog operation: Update time: "2026-05-26T19:58:56Z" name: kuadrant-system.18b335e4128fa38f namespace: default resourceVersion: "17560" uid: ae198a44-219f-41ea-badc-ceffec4ae0c5 reason: ResolutionFailed reportingComponent: operator-lifecycle-manager reportingInstance: "" source: component: operator-lifecycle-manager type: Warning - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Received signal to terminate, becoming unready, but keeping serving metadata: creationTimestamp: "2026-05-26T19:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-26T19:49:41Z" name: kube-system.18b33563fb734fc8 namespace: default resourceVersion: "7743" uid: c14999ad-873a-46ba-8004-69c67d757f35 reason: TerminationStart reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-547cfd84dc-lbq74 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: The minimal shutdown duration of 15s finished metadata: creationTimestamp: "2026-05-26T19:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-26T19:49:56Z" name: kube-system.18b3356779cb3ac4 namespace: default resourceVersion: "7992" uid: 2b3a65c5-1862-49a2-b18a-70887c19cfab reason: TerminationMinimalShutdownDurationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-547cfd84dc-lbq74 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Server has stopped listening metadata: creationTimestamp: "2026-05-26T19:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-26T19:49:56Z" name: kube-system.18b335677a20fe0d namespace: default resourceVersion: "7993" uid: 23d93af7-7d5c-46aa-9941-a7df4116a831 reason: TerminationStoppedServing reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-547cfd84dc-lbq74 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pre-shutdown hooks have been finished metadata: creationTimestamp: "2026-05-26T19:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-26T19:49:56Z" name: kube-system.18b335677a5f4641 namespace: default resourceVersion: "7994" uid: 5d5edb24-13e8-4e77-8e79-6ff8a89dbe37 reason: TerminationPreShutdownHooksFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-547cfd84dc-lbq74 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pending requests processed metadata: creationTimestamp: "2026-05-26T19:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-26T19:50:56Z" name: kube-system.18b33575727dd174 namespace: default resourceVersion: "10360" uid: a212b321-365b-480c-bf49-0d03687afbec reason: TerminationGracefulTerminationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-547cfd84dc-lbq74 type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T19:42:43Z" involvedObject: apiVersion: v1 kind: Namespace name: openshift-kube-apiserver namespace: default kind: Event lastTimestamp: "2026-05-26T19:42:43Z" message: readyz=true metadata: creationTimestamp: "2026-05-26T19:42:43Z" name: openshift-kube-apiserver.18b335029e2b826e namespace: default resourceVersion: "274" uid: 1bfef13d-ab76-4850-916c-e7bed72e9e73 reason: KubeAPIReadyz reportingComponent: "" reportingInstance: "" source: component: apiserver host: kube-apiserver-7746b8cff5-td7rn type: Warning kind: EventList metadata: resourceVersion: "47265"