--- apiVersion: v1 items: - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-28T18:33:25Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Auth name: auth resourceVersion: "14386" uid: 53e83e1e-a1f2-40d7-9af1-b6b8a69e0e9c kind: Event lastTimestamp: "2026-05-28T18:33:32Z" message: 'failure deploying resource {map[apiVersion:rbac.authorization.k8s.io/v1 kind:Role metadata:map[annotations:map[platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:auth platform.opendatahub.io/instance.uid:53e83e1e-a1f2-40d7-9af1-b6b8a69e0e9c platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1] labels:map[platform.opendatahub.io/part-of:auth] name:data-science-admingroup-role namespace:opendatahub ownerReferences:[map[apiVersion:services.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Auth name:auth uid:53e83e1e-a1f2-40d7-9af1-b6b8a69e0e9c]]] rules:[map[apiGroups:[services.platform.opendatahub.io] resources:[auths] verbs:[get list watch patch update]] map[apiGroups:[services.opendatahub.io] resources:[auths/status] verbs:[get]] map[apiGroups:[infrastructure.opendatahub.io] resources:[hardwareprofiles] verbs:[create get list watch patch update delete]] map[apiGroups:[route.openshift.io] resources:[routes] verbs:[get list watch]] map[apiGroups:[batch] resources:[cronjobs] verbs:[get update watch]] map[apiGroups:[image.openshift.io] resources:[imagestreams] verbs:[create get list patch update delete watch]] map[apiGroups:[build.openshift.io] resources:[builds buildconfigs] verbs:[list get watch]] map[apiGroups:[apps] resources:[deployments] verbs:[patch update]] map[apiGroups:[opendatahub.io] resources:[odhdashboardconfigs] verbs:[get list watch create update patch]] map[apiGroups:[dashboard.opendatahub.io] resources:[odhapplications odhdocuments] verbs:[get list watch]] map[apiGroups:[console.openshift.io] resources:[odhquickstarts] verbs:[get list watch]] map[apiGroups:[template.openshift.io] resources:[templates] verbs:[get list watch create patch update delete]] map[apiGroups:[serving.kserve.io] resources:[servingruntimes] verbs:[create]] map[apiGroups:[nim.opendatahub.io] resources:[accounts] verbs:[watch update get list create patch delete]] map[apiGroups:[] resourceNames:[tier-to-group-mapping] resources:[configmaps] verbs:[get list watch patch update]]]]}: apply failed rbac.authorization.k8s.io/v1, Kind=Role: unable to patch rbac.authorization.k8s.io/v1, Kind=Role opendatahub/data-science-admingroup-role: roles.rbac.authorization.k8s.io "data-science-admingroup-role" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion services.platform.opendatahub.io/v1alpha1 Kind Auth: no matches for kind "Auth" in version "services.platform.opendatahub.io/v1alpha1"' metadata: creationTimestamp: "2026-05-28T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:33:32Z" name: auth.18b3ce63cc551171 namespace: default resourceVersion: "14557" uid: e4deb453-c0cf-459a-9d1b-0a619f9e91f7 reason: ProvisioningError reportingComponent: auth reportingInstance: "" source: component: auth type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:32Z" involvedObject: kind: CertificateSigningRequest name: csr-475ft kind: Event lastTimestamp: "2026-05-28T18:26:32Z" message: CSR "csr-475ft" has been approved metadata: creationTimestamp: "2026-05-28T18:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:26:32Z" name: csr-475ft.18b3ce03afecd2cb namespace: default resourceVersion: "6748" uid: f4ca37dc-4934-4f60-9163-c7784d88ae53 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:35Z" involvedObject: kind: CertificateSigningRequest name: csr-8xvgg kind: Event lastTimestamp: "2026-05-28T18:26:35Z" message: CSR "csr-8xvgg" has been approved metadata: creationTimestamp: "2026-05-28T18:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:26:35Z" name: csr-8xvgg.18b3ce045d4aa3c8 namespace: default resourceVersion: "6817" uid: 4e663e7c-2bd4-4325-8011-c4cb33e73f0c reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:25:54Z" involvedObject: kind: CertificateSigningRequest name: csr-99w6k kind: Event lastTimestamp: "2026-05-28T18:25:54Z" message: CSR "csr-99w6k" has been approved metadata: creationTimestamp: "2026-05-28T18:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:25:54Z" name: csr-99w6k.18b3cdfacd5af1fe namespace: default resourceVersion: "5781" uid: 0fde9a32-dfd7-460a-a24c-5d6fb21dcda9 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:42Z" involvedObject: kind: CertificateSigningRequest name: csr-gq87f kind: Event lastTimestamp: "2026-05-28T18:26:42Z" message: CSR "csr-gq87f" has been approved metadata: creationTimestamp: "2026-05-28T18:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:26:42Z" name: csr-gq87f.18b3ce05debf2552 namespace: default resourceVersion: "6921" uid: abff2eb6-6998-4499-a680-7952a9f20069 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:27Z" involvedObject: kind: CertificateSigningRequest name: csr-pp6bm kind: Event lastTimestamp: "2026-05-28T18:26:27Z" message: CSR "csr-pp6bm" has been approved metadata: creationTimestamp: "2026-05-28T18:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:26:27Z" name: csr-pp6bm.18b3ce02800db301 namespace: default resourceVersion: "6611" uid: c82df52a-1246-44aa-aa0a-d8040671fac9 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:01Z" involvedObject: kind: CertificateSigningRequest name: csr-thvt2 kind: Event lastTimestamp: "2026-05-28T18:26:01Z" message: CSR "csr-thvt2" has been approved metadata: creationTimestamp: "2026-05-28T18:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ovnkube-identity operation: Update time: "2026-05-28T18:26:01Z" name: csr-thvt2.18b3cdfc4ff66c57 namespace: default resourceVersion: "5916" uid: 961f7fca-5d16-4f0f-8d65-7ffe9b907f38 reason: CSRApproved reportingComponent: ovnkube-csr-approver-controller reportingInstance: "" source: component: ovnkube-csr-approver-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-28T18:33:25Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14328" uid: 26772193-a388-4028-bbb9-3fc4fb77df15 kind: Event lastTimestamp: "2026-05-28T18:33:31Z" message: 'failed to create auth proxy secret: secrets "kube-auth-proxy-creds" is forbidden: cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion services.platform.opendatahub.io/v1alpha1 Kind GatewayConfig: no matches for kind "GatewayConfig" in version "services.platform.opendatahub.io/v1alpha1"' metadata: creationTimestamp: "2026-05-28T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:33:31Z" name: default-gateway.18b3ce63c189d76d namespace: default resourceVersion: "14553" uid: f4696b8b-a1d0-4e4a-9d1a-c99a908ce895 reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:33:36Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14724" uid: 26772193-a388-4028-bbb9-3fc4fb77df15 kind: Event lastTimestamp: "2026-05-28T18:33:36Z" message: 'failed to create OAuth client: failed to get auth proxy secret openshift-ingress/kube-auth-proxy-creds: Secret "kube-auth-proxy-creds" not found' metadata: creationTimestamp: "2026-05-28T18:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:33:36Z" name: default-gateway.18b3ce665dc2ab33 namespace: default resourceVersion: "14725" uid: 47a79fb0-62aa-44f6-9b33-f0457e5567be reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-28T18:33:37Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: GatewayConfig name: default-gateway resourceVersion: "14784" uid: 26772193-a388-4028-bbb9-3fc4fb77df15 kind: Event lastTimestamp: "2026-05-28T18:33:47Z" message: 'failed to lookup object openshift-ingress/data-science-tls-rule: no matches for kind "DestinationRule" in version "networking.istio.io/v1"' metadata: creationTimestamp: "2026-05-28T18:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:33:47Z" name: default-gateway.18b3ce667b68d709 namespace: default resourceVersion: "14966" uid: 31afef9b-ced9-4273-a62b-f37866bd80b4 reason: ProvisioningError reportingComponent: gatewayconfig reportingInstance: "" source: component: gatewayconfig type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-28T18:33:35Z" involvedObject: apiVersion: components.platform.opendatahub.io/v1alpha1 kind: Kserve name: default-kserve resourceVersion: "14701" uid: a9d548ab-41c9-4256-892e-5b8c626c82cd kind: Event lastTimestamp: "2026-05-28T18:34:02Z" message: "failure deploying resource {map[apiVersion:serving.kserve.io/v1alpha1 kind:LLMInferenceServiceConfig metadata:map[annotations:map[internal.config.kubernetes.io/previousKinds:LLMInferenceServiceConfig internal.config.kubernetes.io/previousNames:kserve-config-llm-decode-template internal.config.kubernetes.io/previousNamespaces:opendatahub platform.opendatahub.io/instance.generation:1 platform.opendatahub.io/instance.name:default-kserve platform.opendatahub.io/instance.uid:a9d548ab-41c9-4256-892e-5b8c626c82cd platform.opendatahub.io/type:Open Data Hub platform.opendatahub.io/version:3.4.0-ea.1 serving.kserve.io/well-known-config:true] labels:map[app.kubernetes.io/part-of:kserve app.opendatahub.io/kserve:true platform.opendatahub.io/part-of:kserve] name:v3-4-0-ea-1-kserve-config-llm-decode-template namespace:opendatahub ownerReferences:[map[apiVersion:components.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:%!s(bool=true) controller:%!s(bool=true) kind:Kserve name:default-kserve uid:a9d548ab-41c9-4256-892e-5b8c626c82cd]]] spec:map[template:map[containers:[map[args:[if [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n cat /proc/driver/nvidia/params\n\n \ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE v2\"}\n\n \ echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*; do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\" ]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\" # Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n \ echo \"[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] && grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" ${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n \ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n \ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\"; do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active HCAs\n \ if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo \"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n \ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n \ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\" 2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n \ # Check for IPv4 GID (contains ffff:)\n if [ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat \"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n \ gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]} + 1))\n break # Use first found IPv4 GID per HCA\n fi\n \ fi\n done\n done\n\n # Find the most common GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n \ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n \ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt $max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n \ fi\n done\n\n # Use deterministic fallback if counts are equal - prefer lower index number \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n \ echo \"[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n # If there's a tie, prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n \ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n \ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)\"\n\n \ export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX\"\n else\n \ echo \"[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX inference.\"\n fi\nfi\n\neval \"vllm serve /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n ${VLLM_ADDITIONAL_ARGS} \\\n --enable-ssl-refresh \\\n --ssl-certfile /var/run/kserve/tls/tls.crt \\\n --ssl-keyfile /var/run/kserve/tls/tls.key\"] command:[/bin/bash -c] env:[map[name:HOME value:/home] map[name:VLLM_LOGGING_LEVEL value:INFO] map[name:HF_HUB_CACHE value:/models]] image:registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=120) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:main ports:[map[containerPort:%!s(int64=8001) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=60) httpGet:map[path:/health port:%!s(int64=8001) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true) seccompProfile:map[type:RuntimeDefault]] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/home name:home] map[mountPath:/dev/shm name:dshm] map[mountPath:/models name:model-cache] map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] initContainers:[map[args:[--port=8000 --vllm-port=8001 --connector=nixlv2 --secure-proxy=true --cert-path=/var/run/kserve/tls --decoder-use-tls=true --prefiller-use-tls=true --enable-ssrf-protection=true --pool-group=inference.networking.x-k8s.io] env:[map[name:INFERENCE_POOL_NAMESPACE valueFrom:map[fieldRef:map[fieldPath:metadata.namespace]]] map[name:SSL_CERT_DIR value:/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs]] image:quay.io/opendatahub/llm-d-routing-sidecar:release-v0.4 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:%!s(int64=3) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=10)] name:llm-d-routing-sidecar ports:[map[containerPort:%!s(int64=8000) protocol:TCP]] readinessProbe:map[failureThreshold:%!s(int64=10) httpGet:map[path:/health port:%!s(int64=8000) scheme:HTTPS] initialDelaySeconds:%!s(int64=10) periodSeconds:%!s(int64=10) timeoutSeconds:%!s(int64=5)] resources:map[] restartPolicy:Always securityContext:map[allowPrivilegeEscalation:%!s(bool=false) capabilities:map[drop:[ALL]] readOnlyRootFilesystem:%!s(bool=false) runAsNonRoot:%!s(bool=true)] terminationMessagePath:/dev/termination-log terminationMessagePolicy:FallbackToLogsOnError volumeMounts:[map[mountPath:/var/run/kserve/tls name:tls-certs readOnly:%!s(bool=true)]]]] terminationGracePeriodSeconds:%!s(int64=30) volumes:[map[emptyDir:map[] name:home] map[emptyDir:map[medium:Memory sizeLimit:1Gi] name:dshm] map[emptyDir:map[] name:model-cache] map[name:tls-certs secret:map[secretName:{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}]]]]]]}: apply failed serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig: unable to patch serving.kserve.io/v1alpha1, Kind=LLMInferenceServiceConfig opendatahub/v3-4-0-ea-1-kserve-config-llm-decode-template: Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.validator\": failed to call webhook: Post \"https://kserve-webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"" metadata: creationTimestamp: "2026-05-28T18:33:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:34:02Z" name: default-kserve.18b3ce6621709d86 namespace: default resourceVersion: "15698" uid: b6764300-4f8c-469e-a928-ca82af5e7396 reason: ProvisioningError reportingComponent: kserve reportingInstance: "" source: component: kserve type: Warning - apiVersion: v1 count: 34 eventTime: null firstTimestamp: "2026-05-28T18:33:25Z" involvedObject: apiVersion: services.platform.opendatahub.io/v1alpha1 kind: Monitoring name: default-monitoring resourceVersion: "14288" uid: 304a2f54-c8b6-4425-996c-3fb5eb472e62 kind: Event lastTimestamp: "2026-05-28T18:50:35Z" message: 'error fetching list of deployments: unable to list: opendatahub-monitoring because of unknown namespace for the cache' metadata: creationTimestamp: "2026-05-28T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-28T18:50:35Z" name: default-monitoring.18b3ce63b9fcfdb8 namespace: default resourceVersion: "36029" uid: e5d19832-cd53-476d-b2a8-73e26a212827 reason: ProvisioningError reportingComponent: monitoring reportingInstance: "" source: component: monitoring type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:25:28Z" involvedObject: kind: Node name: ip-10-0-133-168.ec2.internal uid: ip-10-0-133-168.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:25:28Z" message: 'Node ip-10-0-133-168.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:25:29Z" name: ip-10-0-133-168.ec2.internal.18b3cdf4bf315c6c namespace: default resourceVersion: "5390" uid: 199ac06a-01df-4698-b585-bd70e08c8736 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:25:28Z" involvedObject: kind: Node name: ip-10-0-133-168.ec2.internal uid: ip-10-0-133-168.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:25:28Z" message: 'Node ip-10-0-133-168.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:25:29Z" name: ip-10-0-133-168.ec2.internal.18b3cdf4bf31a8ee namespace: default resourceVersion: "5391" uid: 92d919c1-c55a-433f-9f22-0df2e2437a9d reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:25:28Z" involvedObject: kind: Node name: ip-10-0-133-168.ec2.internal uid: ip-10-0-133-168.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:25:28Z" message: 'Node ip-10-0-133-168.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:25:29Z" name: ip-10-0-133-168.ec2.internal.18b3cdf4bf31d68e namespace: default resourceVersion: "5394" uid: 29e001ae-606a-42be-9d36-fe7695df7d67 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:25:28Z" involvedObject: kind: Node name: ip-10-0-133-168.ec2.internal uid: ip-10-0-133-168.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:25:28Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T18:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:25:28Z" name: ip-10-0-133-168.ec2.internal.18b3cdf4c26ee2ac namespace: default resourceVersion: "5353" uid: a5712e2b-3a76-4913-ba7b-617c42dc8f59 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:25:29Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-133-168.ec2.internal resourceVersion: "5354" uid: ed3c25ea-48c8-4002-9e97-3b3937d4fed1 kind: Event lastTimestamp: "2026-05-28T18:25:29Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T18:25:29Z" name: ip-10-0-133-168.ec2.internal.18b3cdf4db75d26d namespace: default resourceVersion: "5437" uid: 41280ef1-36e5-4112-a0fb-16c5bfcfe0f7 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:25:33Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-133-168.ec2.internal uid: ed3c25ea-48c8-4002-9e97-3b3937d4fed1 kind: Event lastTimestamp: "2026-05-28T18:25:33Z" message: 'Node ip-10-0-133-168.ec2.internal event: Registered Node ip-10-0-133-168.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:25:33Z" name: ip-10-0-133-168.ec2.internal.18b3cdf5ef3ac0bd namespace: default resourceVersion: "5582" uid: ee27ac75-1008-4a51-8d0b-fd3b9694d4fd reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:25:56Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-133-168.ec2.internal resourceVersion: "5794" uid: ed3c25ea-48c8-4002-9e97-3b3937d4fed1 kind: Event lastTimestamp: "2026-05-28T18:25:56Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-133-168.ec2.internal, error getting gateway config for node ip-10-0-133-168.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-133-168.ec2.internal", failed to update chassis to local for local node ip-10-0-133-168.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-133-168.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-133-168.ec2.internal]' metadata: creationTimestamp: "2026-05-28T18:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-133-168 operation: Update time: "2026-05-28T18:25:56Z" name: ip-10-0-133-168.ec2.internal.18b3cdfb2a8e497f namespace: default resourceVersion: "5795" uid: 20c33210-4167-4364-996e-83d3a248da39 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:02Z" involvedObject: kind: Node name: ip-10-0-133-168.ec2.internal uid: ip-10-0-133-168.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:02Z" message: 'Node ip-10-0-133-168.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T18:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:02Z" name: ip-10-0-133-168.ec2.internal.18b3cdfca5135fa0 namespace: default resourceVersion: "5984" uid: a10b25fb-3785-4c06-b46b-2ad3d65b816d reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:28:25Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-133-168.ec2.internal uid: ed3c25ea-48c8-4002-9e97-3b3937d4fed1 kind: Event lastTimestamp: "2026-05-28T18:28:25Z" message: 'Node ip-10-0-133-168.ec2.internal event: Registered Node ip-10-0-133-168.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:28:25Z" name: ip-10-0-133-168.ec2.internal.18b3ce1de3bb5976 namespace: default resourceVersion: "8076" uid: 90cd85cc-8c2d-4311-84d8-72ec324ddf98 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe3007b307 namespace: default resourceVersion: "6143" uid: 931112cc-1224-4dff-8b1e-4ba17ef88bbc reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: 'Node ip-10-0-137-19.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe31fdd953 namespace: default resourceVersion: "6237" uid: 8b996052-edd2-4839-b568-125159e71fdb reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: 'Node ip-10-0-137-19.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe31fe1d0c namespace: default resourceVersion: "6244" uid: cf664709-4425-4b14-8b0e-21ecb3706170 reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: 'Node ip-10-0-137-19.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe31fe413e namespace: default resourceVersion: "6252" uid: 4d4b5a95-20a9-411e-8860-38a13c0fbf3d reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe3492cca5 namespace: default resourceVersion: "6147" uid: 6f3757ab-bf05-401b-aca9-6d804f7e7519 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:09Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-19.ec2.internal resourceVersion: "6149" uid: ea34f9d2-7d4a-47fb-bb41-4dcc4220b711 kind: Event lastTimestamp: "2026-05-28T18:26:09Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T18:26:09Z" name: ip-10-0-137-19.ec2.internal.18b3cdfe4ab0ecad namespace: default resourceVersion: "6253" uid: f7883e27-bdb6-4b30-b5e9-a7ff2ecb9866 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:13Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-19.ec2.internal uid: ea34f9d2-7d4a-47fb-bb41-4dcc4220b711 kind: Event lastTimestamp: "2026-05-28T18:26:13Z" message: 'Node ip-10-0-137-19.ec2.internal event: Registered Node ip-10-0-137-19.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:26:13Z" name: ip-10-0-137-19.ec2.internal.18b3cdff3fd6f983 namespace: default resourceVersion: "6371" uid: b7e3626f-10fd-4778-9321-5609472c2ae4 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:37Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-19.ec2.internal resourceVersion: "6837" uid: ea34f9d2-7d4a-47fb-bb41-4dcc4220b711 kind: Event lastTimestamp: "2026-05-28T18:26:37Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-19.ec2.internal, error getting gateway config for node ip-10-0-137-19.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-137-19.ec2.internal", failed to update chassis to local for local node ip-10-0-137-19.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-137-19.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-137-19.ec2.internal]' metadata: creationTimestamp: "2026-05-28T18:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-137-19 operation: Update time: "2026-05-28T18:26:37Z" name: ip-10-0-137-19.ec2.internal.18b3ce04b98da7ee namespace: default resourceVersion: "6841" uid: e986876d-afb7-4eb4-848c-e3b71c334ae1 reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:43Z" involvedObject: kind: Node name: ip-10-0-137-19.ec2.internal uid: ip-10-0-137-19.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:43Z" message: 'Node ip-10-0-137-19.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T18:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:43Z" name: ip-10-0-137-19.ec2.internal.18b3ce062bd7297c namespace: default resourceVersion: "6932" uid: a23ca763-d870-4e89-ba1c-ee8a86f8c94a reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-137-19.ec2.internal source: component: kubelet host: ip-10-0-137-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:28:25Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-137-19.ec2.internal uid: ea34f9d2-7d4a-47fb-bb41-4dcc4220b711 kind: Event lastTimestamp: "2026-05-28T18:28:25Z" message: 'Node ip-10-0-137-19.ec2.internal event: Registered Node ip-10-0-137-19.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:28:25Z" name: ip-10-0-137-19.ec2.internal.18b3ce1de3bc9a90 namespace: default resourceVersion: "8088" uid: 1f7a0ac2-4cf5-4c7e-9497-bc2da78d11a0 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:00Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:00Z" message: Starting kubelet. metadata: creationTimestamp: "2026-05-28T18:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:00Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc3175b413 namespace: default resourceVersion: "5840" uid: 74779b58-2331-42eb-9459-963acdab0086 reason: Starting reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:00Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:01Z" message: 'Node ip-10-0-140-205.ec2.internal status is now: NodeHasSufficientMemory' metadata: creationTimestamp: "2026-05-28T18:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:01Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc33734257 namespace: default resourceVersion: "5884" uid: 93694aa2-55cf-4434-94a6-56c225b70d36 reason: NodeHasSufficientMemory reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:00Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:01Z" message: 'Node ip-10-0-140-205.ec2.internal status is now: NodeHasNoDiskPressure' metadata: creationTimestamp: "2026-05-28T18:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:01Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc33738259 namespace: default resourceVersion: "5886" uid: f3de973c-753d-4406-be29-56bb2c6cac0e reason: NodeHasNoDiskPressure reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-28T18:26:00Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:01Z" message: 'Node ip-10-0-140-205.ec2.internal status is now: NodeHasSufficientPID' metadata: creationTimestamp: "2026-05-28T18:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:01Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc3373a5ee namespace: default resourceVersion: "5887" uid: 455ee81e-e937-48f6-8797-a6b76963fdb1 reason: NodeHasSufficientPID reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:00Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:00Z" message: Updated Node Allocatable limit across pods metadata: creationTimestamp: "2026-05-28T18:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:00Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc36824ded namespace: default resourceVersion: "5845" uid: 2f330246-5796-4bc2-ad52-38ea3b6088d6 reason: NodeAllocatableEnforced reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:01Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-140-205.ec2.internal resourceVersion: "5848" uid: d5ea6d05-bbc9-435a-8b7e-3a980155d318 kind: Event lastTimestamp: "2026-05-28T18:26:01Z" message: Node synced successfully metadata: creationTimestamp: "2026-05-28T18:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: aws-cloud-controller-manager operation: Update time: "2026-05-28T18:26:01Z" name: ip-10-0-140-205.ec2.internal.18b3cdfc4b81bc36 namespace: default resourceVersion: "5906" uid: 17b43136-12c3-45ab-9abe-bdb8831a50e9 reason: Synced reportingComponent: cloud-node-controller reportingInstance: "" source: component: cloud-node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:03Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-140-205.ec2.internal uid: d5ea6d05-bbc9-435a-8b7e-3a980155d318 kind: Event lastTimestamp: "2026-05-28T18:26:03Z" message: 'Node ip-10-0-140-205.ec2.internal event: Registered Node ip-10-0-140-205.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:26:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:26:03Z" name: ip-10-0-140-205.ec2.internal.18b3cdfceba7eb41 namespace: default resourceVersion: "6090" uid: 3ac59137-5571-4149-9f58-d49eaf6b80d7 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:29Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-140-205.ec2.internal resourceVersion: "6630" uid: d5ea6d05-bbc9-435a-8b7e-3a980155d318 kind: Event lastTimestamp: "2026-05-28T18:26:29Z" message: '[k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-140-205.ec2.internal, error getting gateway config for node ip-10-0-140-205.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-140-205.ec2.internal", failed to update chassis to local for local node ip-10-0-140-205.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-140-205.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-140-205.ec2.internal]' metadata: creationTimestamp: "2026-05-28T18:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: ip-10-0-140-205 operation: Update time: "2026-05-28T18:26:29Z" name: ip-10-0-140-205.ec2.internal.18b3ce02db68acd2 namespace: default resourceVersion: "6632" uid: bc5868f1-880c-4915-a68c-22cdf02e8aab reason: ErrorAddingResource reportingComponent: ovnk-controlplane reportingInstance: "" source: component: ovnk-controlplane type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:26:34Z" involvedObject: kind: Node name: ip-10-0-140-205.ec2.internal uid: ip-10-0-140-205.ec2.internal kind: Event lastTimestamp: "2026-05-28T18:26:34Z" message: 'Node ip-10-0-140-205.ec2.internal status is now: NodeReady' metadata: creationTimestamp: "2026-05-28T18:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-28T18:26:34Z" name: ip-10-0-140-205.ec2.internal.18b3ce03f784fc79 namespace: default resourceVersion: "6769" uid: 19538496-5798-4af4-b854-53040fd42920 reason: NodeReady reportingComponent: kubelet reportingInstance: ip-10-0-140-205.ec2.internal source: component: kubelet host: ip-10-0-140-205.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:28:25Z" involvedObject: apiVersion: v1 kind: Node name: ip-10-0-140-205.ec2.internal uid: d5ea6d05-bbc9-435a-8b7e-3a980155d318 kind: Event lastTimestamp: "2026-05-28T18:28:25Z" message: 'Node ip-10-0-140-205.ec2.internal event: Registered Node ip-10-0-140-205.ec2.internal in Controller' metadata: creationTimestamp: "2026-05-28T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-28T18:28:25Z" name: ip-10-0-140-205.ec2.internal.18b3ce1de3bcd000 namespace: default resourceVersion: "8095" uid: a7e0c0fc-ecfd-4366-95da-5d3f9362f4a2 reason: RegisteredNode reportingComponent: node-controller reportingInstance: "" source: component: node-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-28T18:34:37Z" involvedObject: apiVersion: v1 kind: Namespace name: kuadrant-system resourceVersion: "16552" uid: 90a858ae-65bb-4059-841c-d5910e2b1509 kind: Event lastTimestamp: "2026-05-28T18:34:41Z" message: 'error using catalogsource kuadrant-system/kuadrant-operator-catalog: failed to list bundles: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"' metadata: creationTimestamp: "2026-05-28T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: catalog operation: Update time: "2026-05-28T18:34:43Z" name: kuadrant-system.18b3ce749628c1de namespace: default resourceVersion: "16815" uid: 5d9c886f-9530-4279-b802-a8b14d883598 reason: ResolutionFailed reportingComponent: operator-lifecycle-manager reportingInstance: "" source: component: operator-lifecycle-manager type: Warning - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Received signal to terminate, becoming unready, but keeping serving metadata: creationTimestamp: "2026-05-28T18:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T18:28:04Z" name: kube-system.18b3ce18fed99380 namespace: default resourceVersion: "7742" uid: 5ad46300-88eb-46d8-9b64-7d2958d11104 reason: TerminationStart reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-bdf469fbb-xzv8p type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: The minimal shutdown duration of 15s finished metadata: creationTimestamp: "2026-05-28T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T18:28:19Z" name: kube-system.18b3ce1c7d65a839 namespace: default resourceVersion: "8021" uid: 07fe5a15-e5dd-47ce-b361-cbbcc68f7070 reason: TerminationMinimalShutdownDurationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-bdf469fbb-xzv8p type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: Server has stopped listening metadata: creationTimestamp: "2026-05-28T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T18:28:19Z" name: kube-system.18b3ce1c7db9f04a namespace: default resourceVersion: "8022" uid: bf015d64-3843-4190-8000-4ed5d45c256b reason: TerminationStoppedServing reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-bdf469fbb-xzv8p type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pre-shutdown hooks have been finished metadata: creationTimestamp: "2026-05-28T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T18:28:19Z" name: kube-system.18b3ce1c7e11ad75 namespace: default resourceVersion: "8023" uid: 8bde0422-806f-46fa-925c-373d74247b4a reason: TerminationPreShutdownHooksFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-bdf469fbb-xzv8p type: Normal - apiVersion: v1 eventTime: null firstTimestamp: null involvedObject: apiVersion: v1 kind: Namespace name: kube-system namespace: default kind: Event lastTimestamp: null message: All pending requests processed metadata: creationTimestamp: "2026-05-28T18:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:involvedObject: {} f:message: {} f:reason: {} f:source: f:component: {} f:host: {} f:type: {} manager: openshift-apiserver operation: Update time: "2026-05-28T18:29:19Z" name: kube-system.18b3ce2a760c3936 namespace: default resourceVersion: "10167" uid: c5fee852-2313-4ba1-940d-4c34e0ab3778 reason: TerminationGracefulTerminationFinished reportingComponent: "" reportingInstance: "" source: component: apiserver host: openshift-apiserver-bdf469fbb-xzv8p type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-28T18:21:04Z" involvedObject: apiVersion: v1 kind: Namespace name: openshift-kube-apiserver namespace: default kind: Event lastTimestamp: "2026-05-28T18:21:04Z" message: readyz=true metadata: creationTimestamp: "2026-05-28T18:21:04Z" name: openshift-kube-apiserver.18b3cdb73d673de6 namespace: default resourceVersion: "274" uid: f60f71d6-020a-4c3d-98f3-2234473ee3b4 reason: KubeAPIReadyz reportingComponent: "" reportingInstance: "" source: component: apiserver host: kube-apiserver-86f8dfbc94-sbxrl type: Warning kind: EventList metadata: resourceVersion: "47768"