--- apiVersion: v1 items: - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T21:21:17Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Auth name: auth resourceVersion: "14633" uid: e4f1388b-4a94-4da8-a22d-395a0961546c kind: Event lastTimestamp: "2026-04-17T21:21:21Z" message: 'failure deploying resource {map[apiVersion:rbac.authorization.k8s.io/v1 kind:Role metadata:map[annotations:map[platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:auth platform.opendatahub.io/instance.uid:e4f1388b-4a94-4da8-a22d-395a0961546c platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1] labels:map[platform.opendatahub.io/part-of:auth] name:data-science-admingroup-role namespace:opendatahub ownerReferences:[map[apiVersion:services.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Auth name:auth uid:e4f1388b-4a94-4da8-a22d-395a0961546c]]] rules:[map[apiGroups:[services.platform.opendatahub.io] resources:[auths] verbs:[get list watch patch update]] map[apiGroups:[services.opendatahub.io] resources:[auths/status] verbs:[get]] map[apiGroups:[infrastructure.opendatahub.io] resources:[hardwareprofiles] verbs:[create get list watch patch update delete]] map[apiGroups:[route.openshift.io] resources:[routes] verbs:[get list watch]] map[apiGroups:[batch] resources:[cronjobs] verbs:[get update watch]] map[apiGroups:[image.openshift.io] resources:[imagestreams] verbs:[create get list patch update delete watch]] map[apiGroups:[build.openshift.io] resources:[builds buildconfigs] verbs:[list get watch]] map[apiGroups:[apps] resources:[deployments] verbs:[patch update]] map[apiGroups:[opendatahub.io] resources:[odhdashboardconfigs] verbs:[get list watch create update patch]] map[apiGroups:[dashboard.opendatahub.io] resources:[odhapplications odhdocuments] verbs:[get list watch]] map[apiGroups:[console.openshift.io] resources:[odhquickstarts] verbs:[get list watch]] map[apiGroups:[template.openshift.io] resources:[templates] verbs:[get list watch create patch update delete]] map[apiGroups:[serving.kserve.io] resources:[servingruntimes] verbs:[create]] map[apiGroups:[nim.opendatahub.io] resources:[accounts] verbs:[watch update get list create patch delete]] map[apiGroups:[] resourceNames:[tier-to-group-mapping] resources:[configmaps] verbs:[get list watch patch update]]]]}: apply failed rbac.authorization.k8s.io/v1, Kind=Role: unable to patch rbac.authorization.k8s.io/v1, Kind=Role opendatahub/data-science-admingroup-role: roles.rbac.authorization.k8s.io "data-science-admingroup-role" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion services.platform.opendatahub.io/v1alpha1 Kind Auth: no matches for kind "Auth" in version "services.platform.opendatahub.io/v1alpha1"' metadata: creationTimestamp: "2026-04-17T21:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T21:21:21Z" name: auth.18a741c18b5f15c9 namespace: default resourceVersion: "14753" uid: b44dcb5e-098e-4208-a13e-53a2ef25b374 reason: ProvisioningError reportingComponent: auth reportingInstance: "" source: component: auth type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:37Z" involvedObject: kind: CertificateSigningRequest name: csr-2kzw8 kind: Event lastTimestamp: "2026-04-17T21:13:37Z" message: CSR "csr-2kzw8" has been approved metadata: creationTimestamp: "2026-04-17T21:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:13:37Z" name: csr-2kzw8.18a741566a949591 namespace: default resourceVersion: "5916" uid: 30df51d4-9bdb-4ee7-8b8a-b0bf6c9a2f46 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:43Z" involvedObject: kind: CertificateSigningRequest name: csr-6cnpf kind: Event lastTimestamp: "2026-04-17T21:13:43Z" message: CSR "csr-6cnpf" has been approved metadata: creationTimestamp: "2026-04-17T21:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:13:43Z" name: csr-6cnpf.18a74157e3fc2ee0 namespace: default resourceVersion: "6041" uid: 57c8f5f2-1929-4f22-8e2e-a9b9e843515a reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:42Z" involvedObject: kind: CertificateSigningRequest name: csr-l2sh7 kind: Event lastTimestamp: "2026-04-17T21:13:42Z" message: CSR "csr-l2sh7" has been approved metadata: creationTimestamp: "2026-04-17T21:13:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:13:42Z" name: csr-l2sh7.18a741577d3285ff namespace: default resourceVersion: "6015" uid: 59320ab8-b5b4-48c0-bde2-d5ebea844dbf reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:47Z" involvedObject: kind: CertificateSigningRequest name: csr-qxsvp kind: Event lastTimestamp: "2026-04-17T21:13:47Z" message: CSR "csr-qxsvp" has been approved metadata: creationTimestamp: "2026-04-17T21:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:13:47Z" name: csr-qxsvp.18a74158b0f87081 namespace: default resourceVersion: "6145" uid: 0bd48483-7618-42ec-aef2-7526cd74c1af reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:37Z" involvedObject: kind: CertificateSigningRequest name: csr-rbqsb kind: Event lastTimestamp: "2026-04-17T21:14:37Z" message: CSR "csr-rbqsb" has been approved metadata: creationTimestamp: "2026-04-17T21:14:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:14:37Z" name: csr-rbqsb.18a741646da43f4f namespace: default resourceVersion: "6849" uid: d35e3493-ec43-4ab7-8d3b-568359c87f52 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:30Z" involvedObject: kind: CertificateSigningRequest name: csr-zsq6r kind: Event lastTimestamp: "2026-04-17T21:14:30Z" message: CSR "csr-zsq6r" has been approved metadata: creationTimestamp: "2026-04-17T21:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-04-17T21:14:30Z" name: csr-zsq6r.18a74162bd8814ac namespace: default resourceVersion: "6776" uid: 32ff8a0a-68b8-4e42-8f8e-33d5ee90af9b reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T21:21:17Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14552" uid: cc71be5c-a923-47bc-9ef3-096b75d5ad09 kind: Event lastTimestamp: "2026-04-17T21:21:21Z" message: 'failed to create auth proxy secret: secrets "kube-auth-proxy-creds" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion services.platform.opendatahub.io/v1alpha1 Kind GatewayConfig: no matches for kind "GatewayConfig" in version "services.platform.opendatahub.io/v1alpha1"' metadata: creationTimestamp: "2026-04-17T21:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T21:21:21Z" name: default-gateway.18a741c17fff9762 namespace: default resourceVersion: "14756" uid: 2ee78f9c-4c25-4288-84b1-38b4923809be reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T21:21:24Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14897" uid: cc71be5c-a923-47bc-9ef3-096b75d5ad09 kind: Event lastTimestamp: "2026-04-17T21:21:29Z" message: 'failed to lookup object openshift-ingress/data-science-tls-rule: no matches for kind "DestinationRule" in version "networking.istio.io/v1"' metadata: creationTimestamp: "2026-04-17T21:21:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T21:21:29Z" name: default-gateway.18a741c330ff4ce3 namespace: default resourceVersion: "15213" uid: 2c65a0f4-1339-4907-a64f-4709ff6677b4 reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-17T21:21:27Z" involvedObject: apiVersion: components.platform.opendatahub.io/v1alpha1 kind: Kserve name: default-kserve resourceVersion: "15088" uid: bf695196-5a51-4010-b9a2-c025b5a37722 kind: Event lastTimestamp: "2026-04-17T21:21:52Z" message: "failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:bf695196-5a51-4010-b9a2-c025b5a37722 platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:bf695196-5a51-4010-b9a2-c025b5a37722]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n \ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n \ echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n \ echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n \ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n \ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n \ if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n \ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n \ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n \ # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n \ gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n \ fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n \ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n \ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n \ fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n \ echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n \ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n \ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n \ export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n \ echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"" metadata: creationTimestamp: "2026-04-17T21:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T21:21:52Z" name: default-kserve.18a741c3e8e55dee namespace: default resourceVersion: "15971" uid: c2ea4a24-0e8f-4a65-afad-01beeb57e168 reason: ProvisioningError reportingComponent: kserve reportingInstance: "" source: component: kserve type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-17T21:21:17Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Monitoring name: default-monitoring resourceVersion: "14518" uid: 57b08f70-cc33-4a9f-bfd1-08092a5ad1e1 kind: Event lastTimestamp: "2026-04-17T21:22:01Z" message: 'error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache' metadata: creationTimestamp: "2026-04-17T21:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T21:22:01Z" name: default-monitoring.18a741c1799c65a9 namespace: default resourceVersion: "16528" uid: 992c89f4-0f84-405c-be74-16e413bffade reason: ProvisioningError reportingComponent: monitoring reportingInstance: "" source: component: monitoring type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:20:24Z" involvedObject: apiVersion: operator.openshift.io/v1alpha1 kind: IstioCSR kind: Event lastTimestamp: "2026-04-17T21:20:24Z" message: controller is starting metadata: creationTimestamp: "2026-04-17T21:20:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: cert-manager-operator operation: Update time: "2026-04-17T21:20:24Z" name: e2a0313b-b146-427c-aec5-8686b450db52 namespace: default resourceVersion: "12126" uid: 73ca19c7-e83f-42e9-b99d-be4ab214cb37 reason: ControllerStarted reportingComponent: cert-manager-istio-csr-controller reportingInstance: "" source: component: cert-manager-istio-csr-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:04Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:04Z" message: Starting kubelet. metadata: creationTimestamp: "2026-04-17T21:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:04Z" name: ip-10-0-134-198.ec2.internal.18a7415cd049ba98 namespace: default resourceVersion: "6331" uid: e548368c-7707-4269-b69c-b6bcd5668887 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:14:04Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:05Z" message: 'Node ip-10-0-134-198.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-04-17T21:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:05Z" name: ip-10-0-134-198.ec2.internal.18a7415cd225ab69 namespace: default resourceVersion: "6375" uid: 64073389-1b51-4e99-9dc2-a1d1ca7b9a4e reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:14:04Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:05Z" message: 'Node ip-10-0-134-198.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-04-17T21:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:05Z" name: ip-10-0-134-198.ec2.internal.18a7415cd225efb7 namespace: default resourceVersion: "6376" uid: 24b6ad77-35ef-454e-b601-99043245fa47 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:14:04Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:05Z" message: 'Node ip-10-0-134-198.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-04-17T21:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:05Z" name: ip-10-0-134-198.ec2.internal.18a7415cd226168d namespace: default resourceVersion: "6377" uid: fd17b886-2480-499a-98c5-941f94f2eb40 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:05Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:05Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-04-17T21:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:05Z" name: ip-10-0-134-198.ec2.internal.18a7415cd63a3096 namespace: default resourceVersion: "6335" uid: 513f2c58-78d9-417a-b758-208d92c8e910 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:05Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-134-198.ec2.internal resourceVersion: "6337" uid: bf1d780a-de00-45f4-ac94-78c460be46dc kind: Event lastTimestamp: "2026-04-17T21:14:05Z" message: Node synced successfully metadata: creationTimestamp: "2026-04-17T21:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-04-17T21:14:05Z" name: ip-10-0-134-198.ec2.internal.18a7415ceb298a6d namespace: default resourceVersion: "6412" uid: dcce04a9-fcb9-47b7-ad18-bb43ff2aa8e2 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:07Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-134-198.ec2.internal uid: bf1d780a-de00-45f4-ac94-78c460be46dc kind: Event lastTimestamp: "2026-04-17T21:14:07Z" message: 'Node ip-10-0-134-198.ec2.internal event: Registered Node ip-10-0-134-198.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:14:07Z" name: ip-10-0-134-198.ec2.internal.18a7415d7ff263ca namespace: default resourceVersion: "6507" uid: 365e379a-94cc-4b4d-b9a5-a1f40a0bd81d reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:31Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-134-198.ec2.internal resourceVersion: "6796" uid: bf1d780a-de00-45f4-ac94-78c460be46dc kind: Event lastTimestamp: "2026-04-17T21:14:31Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-134-198.ec2.internal, error getting gateway config for node ip-10-0-134-198.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-134-198.ec2.internal", failed to update chassis to local for local node ip-10-0-134-198.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-134-198.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-134-198.ec2.internal]' metadata: creationTimestamp: "2026-04-17T21:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-134-198 operation: Update time: "2026-04-17T21:14:31Z" name: ip-10-0-134-198.ec2.internal.18a74163192071aa namespace: default resourceVersion: "6799" uid: 8db16c57-8f77-40ca-8ff5-019e2004ceb3 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:14:39Z" involvedObject: kind: Node name: ip-10-0-134-198.ec2.internal uid: ip-10-0-134-198.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:14:39Z" message: 'Node ip-10-0-134-198.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-04-17T21:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:14:39Z" name: ip-10-0-134-198.ec2.internal.18a74164d1065ee6 namespace: default resourceVersion: "6863" uid: 1d9bbf20-ce71-4f6f-8749-d41d91029286 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-134-198.ec2.internal source: component: kubelet host: ip-10-0-134-198.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:16:05Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-134-198.ec2.internal uid: bf1d780a-de00-45f4-ac94-78c460be46dc kind: Event lastTimestamp: "2026-04-17T21:16:05Z" message: 'Node ip-10-0-134-198.ec2.internal event: Registered Node ip-10-0-134-198.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:16:05Z" name: ip-10-0-134-198.ec2.internal.18a74178eb4c2ac3 namespace: default resourceVersion: "7980" uid: acf6fc48-93d8-4bed-a4d1-cbe82889041d reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:16Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:16Z" message: Starting kubelet. metadata: creationTimestamp: "2026-04-17T21:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:16Z" name: ip-10-0-135-174.ec2.internal.18a7415197a22ac7 namespace: default resourceVersion: "5438" uid: 7d5b357e-83c4-4a6a-add1-ced543eca567 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T21:13:16Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:16Z" message: 'Node ip-10-0-135-174.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-04-17T21:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:16Z" name: ip-10-0-135-174.ec2.internal.18a7415199415bb3 namespace: default resourceVersion: "5443" uid: 8e3bd8f5-238e-40b7-9ff3-7ca313c7736f reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T21:13:16Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:16Z" message: 'Node ip-10-0-135-174.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-04-17T21:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:16Z" name: ip-10-0-135-174.ec2.internal.18a7415199419a72 namespace: default resourceVersion: "5445" uid: 3f64a4e4-1f93-401a-a4f4-cfd9677d3d42 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T21:13:16Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:16Z" message: 'Node ip-10-0-135-174.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-04-17T21:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:16Z" name: ip-10-0-135-174.ec2.internal.18a741519941bc74 namespace: default resourceVersion: "5455" uid: 0cc29b8b-4def-453a-b54c-402c3b357e5d reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:16Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:16Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-04-17T21:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:16Z" name: ip-10-0-135-174.ec2.internal.18a741519bb7245f namespace: default resourceVersion: "5442" uid: 9793fe6d-7452-42e5-8e58-d40fe321a7bc reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:17Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-135-174.ec2.internal resourceVersion: "5444" uid: 7e964740-45c9-419d-a45c-99b565b74338 kind: Event lastTimestamp: "2026-04-17T21:13:17Z" message: Node synced successfully metadata: creationTimestamp: "2026-04-17T21:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-04-17T21:13:17Z" name: ip-10-0-135-174.ec2.internal.18a74151b1279bd4 namespace: default resourceVersion: "5533" uid: d7cfea61-e717-44fa-a614-60bae3bb0e7e reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:17Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-135-174.ec2.internal uid: 7e964740-45c9-419d-a45c-99b565b74338 kind: Event lastTimestamp: "2026-04-17T21:13:17Z" message: 'Node ip-10-0-135-174.ec2.internal event: Registered Node ip-10-0-135-174.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:13:17Z" name: ip-10-0-135-174.ec2.internal.18a74151db5a8eb5 namespace: default resourceVersion: "5566" uid: e565ecb5-85ad-4b1f-a2ae-fd1d862f2988 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:43Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-135-174.ec2.internal resourceVersion: "6035" uid: 7e964740-45c9-419d-a45c-99b565b74338 kind: Event lastTimestamp: "2026-04-17T21:13:43Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-135-174.ec2.internal, error getting gateway config for node ip-10-0-135-174.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-135-174.ec2.internal", failed to update chassis to local for local node ip-10-0-135-174.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-135-174.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-135-174.ec2.internal]' metadata: creationTimestamp: "2026-04-17T21:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-135-174 operation: Update time: "2026-04-17T21:13:43Z" name: ip-10-0-135-174.ec2.internal.18a74157da1f71db namespace: default resourceVersion: "6037" uid: 8496d3c1-c8a5-4c48-87d7-d6e5fe358d17 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:48Z" involvedObject: kind: Node name: ip-10-0-135-174.ec2.internal uid: ip-10-0-135-174.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:48Z" message: 'Node ip-10-0-135-174.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-04-17T21:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:48Z" name: ip-10-0-135-174.ec2.internal.18a7415904c3a9f7 namespace: default resourceVersion: "6162" uid: 6bbf3c2e-393f-4849-87ac-168e4d68124a reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:16:05Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-135-174.ec2.internal uid: 7e964740-45c9-419d-a45c-99b565b74338 kind: Event lastTimestamp: "2026-04-17T21:16:05Z" message: 'Node ip-10-0-135-174.ec2.internal event: Registered Node ip-10-0-135-174.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:16:05Z" name: ip-10-0-135-174.ec2.internal.18a74178eb4d31d1 namespace: default resourceVersion: "7985" uid: 5d4d8675-f9c4-4601-8882-e7edafb2d2c4 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:13:09Z" involvedObject: kind: Node name: ip-10-0-138-36.ec2.internal uid: ip-10-0-138-36.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:09Z" message: 'Node ip-10-0-138-36.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-04-17T21:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:09Z" name: ip-10-0-138-36.ec2.internal.18a7414fdbbd69b8 namespace: default resourceVersion: "5249" uid: 68b0c1bb-c172-4bae-a337-ef3f54eb53b3 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-138-36.ec2.internal source: component: kubelet host: ip-10-0-138-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:13:09Z" involvedObject: kind: Node name: ip-10-0-138-36.ec2.internal uid: ip-10-0-138-36.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:09Z" message: 'Node ip-10-0-138-36.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-04-17T21:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:09Z" name: ip-10-0-138-36.ec2.internal.18a7414fdbbdcc5d namespace: default resourceVersion: "5258" uid: 8550a3d6-5fa5-41ee-8921-c50ca75fd0b2 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-138-36.ec2.internal source: component: kubelet host: ip-10-0-138-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T21:13:09Z" involvedObject: kind: Node name: ip-10-0-138-36.ec2.internal uid: ip-10-0-138-36.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:09Z" message: 'Node ip-10-0-138-36.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-04-17T21:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:09Z" name: ip-10-0-138-36.ec2.internal.18a7414fdbbdf7a5 namespace: default resourceVersion: "5260" uid: 54166468-e21c-4b42-b9b7-00a78d0234d6 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-138-36.ec2.internal source: component: kubelet host: ip-10-0-138-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:09Z" involvedObject: kind: Node name: ip-10-0-138-36.ec2.internal uid: ip-10-0-138-36.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:09Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-04-17T21:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:09Z" name: ip-10-0-138-36.ec2.internal.18a7414fdee2295b namespace: default resourceVersion: "5219" uid: fc465819-9985-4b64-92ec-26f17b17e4df reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-138-36.ec2.internal source: component: kubelet host: ip-10-0-138-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:09Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-138-36.ec2.internal resourceVersion: "5220" uid: 89af94e1-db7c-4348-9437-600fc400e46d kind: Event lastTimestamp: "2026-04-17T21:13:09Z" message: Node synced successfully metadata: creationTimestamp: "2026-04-17T21:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-04-17T21:13:09Z" name: ip-10-0-138-36.ec2.internal.18a7414ff52bd73c namespace: default resourceVersion: "5292" uid: 1ad018e4-c3b6-4e03-87ef-70b49d37d50c reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:12Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-138-36.ec2.internal uid: 89af94e1-db7c-4348-9437-600fc400e46d kind: Event lastTimestamp: "2026-04-17T21:13:12Z" message: 'Node ip-10-0-138-36.ec2.internal event: Registered Node ip-10-0-138-36.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:13:12Z" name: ip-10-0-138-36.ec2.internal.18a74150b145f026 namespace: default resourceVersion: "5401" uid: f02856ec-05f0-4520-b9fa-789cadf31b15 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:38Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-138-36.ec2.internal resourceVersion: "5941" uid: 89af94e1-db7c-4348-9437-600fc400e46d kind: Event lastTimestamp: "2026-04-17T21:13:38Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-138-36.ec2.internal, error getting gateway config for node ip-10-0-138-36.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-138-36.ec2.internal", failed to update chassis to local for local node ip-10-0-138-36.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-138-36.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-138-36.ec2.internal]' metadata: creationTimestamp: "2026-04-17T21:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-138-36 operation: Update time: "2026-04-17T21:13:38Z" name: ip-10-0-138-36.ec2.internal.18a74156c6bdf44c namespace: default resourceVersion: "5942" uid: 336f8ebe-e140-432a-a193-ddfc7284a542 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:13:45Z" involvedObject: kind: Node name: ip-10-0-138-36.ec2.internal uid: ip-10-0-138-36.ec2.internal kind: Event lastTimestamp: "2026-04-17T21:13:45Z" message: 'Node ip-10-0-138-36.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-04-17T21:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T21:13:45Z" name: ip-10-0-138-36.ec2.internal.18a7415835880f42 namespace: default resourceVersion: "6068" uid: 04372219-d6a5-41cf-a605-12bae71262b8 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-138-36.ec2.internal source: component: kubelet host: ip-10-0-138-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:16:05Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-138-36.ec2.internal uid: 89af94e1-db7c-4348-9437-600fc400e46d kind: Event lastTimestamp: "2026-04-17T21:16:05Z" message: 'Node ip-10-0-138-36.ec2.internal event: Registered Node ip-10-0-138-36.ec2.internal in Controller' metadata: creationTimestamp: "2026-04-17T21:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:16:05Z" name: ip-10-0-138-36.ec2.internal.18a74178eb4d5021 namespace: default resourceVersion: "7987" uid: 217ad478-1d42-4315-99ba-1328ea8a1f7d reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T21:22:29Z" involvedObject: apiVersion: v1 kind: Namespace name: kuadrant-system resourceVersion: "16831" uid: 1156c8e5-8186-424a-8194-f441c6a487f9 kind: Event lastTimestamp: "2026-04-17T21:22:32Z" message: 'error using catalogsource kuadrant-system/kuadrant-operator-catalog: failed to list bundles: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"' metadata: creationTimestamp: "2026-04-17T21:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: catalog operation: Update time: "2026-04-17T21:22:33Z" name: kuadrant-system.18a741d24eca1e5e namespace: default resourceVersion: "17056" uid: 804bb5fe-501a-423d-81ad-f3c5dd62dedf reason: ResolutionFailed reportingComponent: operator-lifecycle-manager reportingInstance: "" source: component: operator-lifecycle-manager type: Warning - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Received signal to terminate, becoming unready, but keeping serving metadata: creationTimestamp: "2026-04-17T21:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-04-17T21:15:47Z" name: kube-system.18a74174a4c953af namespace: default resourceVersion: "7595" uid: bbd64577-35ea-4bc0-a2d7-c234d0157eb7 reason: TerminationStart reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-c4bddf8d9-rmq66 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: The minimal shutdown duration of 15s finished metadata: creationTimestamp: "2026-04-17T21:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-04-17T21:16:02Z" name: kube-system.18a741782383a04a namespace: default resourceVersion: "7879" uid: 46fe05dc-a736-4d85-89b1-932c79618e80 reason: TerminationMinimalShutdownDurationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-c4bddf8d9-rmq66 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Server has stopped listening metadata: creationTimestamp: "2026-04-17T21:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-04-17T21:16:02Z" name: kube-system.18a7417823bca5b7 namespace: default resourceVersion: "7880" uid: 5368490b-a05c-4f82-89d4-c306e09a233d reason: TerminationStoppedServing reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-c4bddf8d9-rmq66 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pre-shutdown hooks have been finished metadata: creationTimestamp: "2026-04-17T21:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-04-17T21:16:02Z" name: kube-system.18a7417823f46758 namespace: default resourceVersion: "7881" uid: 6a493562-2b3a-47f8-bf4f-9f84863b2925 reason: TerminationPreShutdownHooksFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-c4bddf8d9-rmq66 type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pending requests processed metadata: creationTimestamp: "2026-04-17T21:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-04-17T21:17:02Z" name: kube-system.18a741861c08c679 namespace: default resourceVersion: "10060" uid: eaf6ad20-5a8a-44c6-bacf-56aa35fa0f04 reason: TerminationGracefulTerminationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-c4bddf8d9-rmq66 type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:08:55Z" involvedObject: apiVersion: v1 kind: Namespace name: openshift-kube-apiserver namespace: default kind: Event lastTimestamp: "2026-04-17T21:08:55Z" message: readyz=true metadata: creationTimestamp: "2026-04-17T21:08:55Z" name: openshift-kube-apiserver.18a74114cff52bb4 namespace: default resourceVersion: "274" uid: 21ed0d94-10dd-496b-9091-7c7ee9d55480 reason: KubeAPIReadyz reportingComponent: "" reportingInstance: "" source: component: apiserver host: kube-apiserver-5c86457896-nsrkt type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T21:25:17Z" involvedObject: apiVersion: v1 kind: Endpoints name: premium-simulated-simulated-premium-kserve-workload-svc kind: Event lastTimestamp: "2026-04-17T21:25:17Z" message: 'Failed to create endpoint for service llm/premium-simulated-simulated-premium-kserve-workload-svc: endpoints "premium-simulated-simulated-premium-kserve-workload-svc" already exists' metadata: creationTimestamp: "2026-04-17T21:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T21:25:17Z" name: premium-simulated-simulated-premium-kserve-workload-svc.18a741f97f1995ad namespace: default resourceVersion: "23030" uid: 7f787420-f725-4623-8b19-21fe3eb6dd7c reason: FailedToCreateEndpoint reportingComponent: endpoint-controller reportingInstance: "" source: component: endpoint-controller type: Warning kind: EventList metadata: resourceVersion: "27517"