--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:14:43.854971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18229" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:14:43Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c70687843893 namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: bc27e01d-0cd0-4935-972b-cb6380c82c62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18239" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: "2026-04-22T20:14:44Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:14:44Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c706a113f7d7 namespace: kserve-ci-e2e-test resourceVersion: "18250" uid: d90e7045-565c-451c-8c9c-f371525d9c09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:48Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c706a268a6c6 namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: 0a93ec38-def6-4955-916a-b2af40a1b561 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:48Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c706a56efa6e namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: 6cac20af-eef7-4b7f-8410-bb9584d6857b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:48Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c706a6596633 namespace: kserve-ci-e2e-test resourceVersion: "18297" uid: bdffe396-6b08-4df9-bb8c-8f63d31eea69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: d2b9873e-d0f0-4974-81f0-069756d60cd6 kind: Event lastTimestamp: "2026-04-22T20:14:49Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-22T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:49Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h.18a8c707c2e7add1 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: 66ca6ec2-de31-4234-af0f-20e8a552a3f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-dcfe87-predictor-5b4466c647 namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: 34ae5efa-3b09-4b58-b2e4-ff4d03dd8b44 kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: 'Created pod: isvc-init-fail-dcfe87-predictor-5b4466c647-ndp8h' metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:43Z" name: isvc-init-fail-dcfe87-predictor-5b4466c647.18a8c70686cd999b namespace: kserve-ci-e2e-test resourceVersion: "18232" uid: f3f3e51e-1932-466d-b781-038c15c0aae7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-dcfe87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18208" uid: 440c0a57-8699-48da-accc-de1c68743bfc kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: Scaled up replica set isvc-init-fail-dcfe87-predictor-5b4466c647 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:43Z" name: isvc-init-fail-dcfe87-predictor.18a8c70683f031a2 namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: 5989b93c-10c5-480a-b888-775246fd086c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-dcfe87 namespace: kserve-ci-e2e-test resourceVersion: "18193" uid: ab71b943-c631-4642-b150-7b7e45420d27 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:48Z" name: isvc-init-fail-dcfe87.18a8c70683c5ec18 namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: e51550f9-1fc4-43bd-aaee-b98f5f235494 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-dcfe87 namespace: kserve-ci-e2e-test resourceVersion: "18193" uid: ab71b943-c631-4642-b150-7b7e45420d27 kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: 'Failed to update status for InferenceService "isvc-init-fail-dcfe87": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-dcfe87": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:43Z" name: isvc-init-fail-dcfe87.18a8c706845250f9 namespace: kserve-ci-e2e-test resourceVersion: "18223" uid: 20b47127-c895-4c82-b43e-3c764c31ab17 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-dcfe87 namespace: kserve-ci-e2e-test resourceVersion: "18193" uid: ab71b943-c631-4642-b150-7b7e45420d27 kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-dcfe87": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:43Z" name: isvc-init-fail-dcfe87.18a8c7068452ac9e namespace: kserve-ci-e2e-test resourceVersion: "18225" uid: 6822c7ff-79f6-4dee-8781-3c600b1ef76c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:09:42.077012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c04425f910 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 248b4d76-8c00-473f-92f7-b4ca3b3b4a32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c05cd394c7 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: eef6b4c6-8f32-42f6-ac6e-205ea9011801 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c05e09a6ff namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: d622ae9b-2b3f-469d-a96c-3c4036c02363 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c060dbfc5f namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 5279594c-f548-482c-9fac-c3c8150de2c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c06146dc0a namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 24d02bf3-6bf9-4dfd-ad41-18ad951a6070 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c14359b051 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 8648735e-b481-418e-8569-b6e3003b64d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c1466ae39c namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 1d366297-3540-4c17-8b3f-816ae667b52a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c146d84b2e namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 45a63799-894b-4a20-9e14-d6d9d0c91952 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:84e96718cb727bdb7a70c88a1b15da870dac18fe5721015f96b729224acb0c47" already present on machine metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c146e35de5 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: cf7358ec-6f3a-4c72-99cb-63761f71cec0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c14997bb85 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: f9a9d1cb-d447-4bc2-8825-911027149d05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:46Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c14a039de3 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: ecd47fc5-3cd0-4acf-95ef-f78146153431 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T20:09:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:11:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:09:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:19Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c17f4d5772 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: bd84c864-ca8b-43b2-9e32-e6454b86c03b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:09:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:11:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:19Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6c1bb07c443 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 24dd6f06-f677-48c8-a996-35f6ee1ae9a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6d1c49ab814 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: 51d5a3d5-9eea-45bb-826c-612271f6ce9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: b67eab5e-b3f4-434c-90e9-aa7ff413b66a kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496.18a8c6d1c49b0b7f namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 0faec011-b3a2-4b63-99aa-09fed2287c8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-d0c94-predictor-84ddff5b74 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: cbcd41fb-bd99-4284-b559-9db22c4f4047 kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: 'Created pod: isvc-logger-raw-d0c94-predictor-84ddff5b74-vs496' metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor-84ddff5b74.18a8c6c043c5198d namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 9eca962f-a6b3-451c-92bd-75586da534a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: 5906ca8d-82ae-4974-826b-da273e4ff496 kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: Scaled up replica set isvc-logger-raw-d0c94-predictor-84ddff5b74 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94-predictor.18a8c6c041034642 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: e55302ae-7e1b-4659-81f0-9c46f23beff1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:09:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: e4c44752-fede-40dd-8615-c0b81ab3ab9d kind: Event lastTimestamp: "2026-04-22T20:10:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:12Z" name: isvc-logger-raw-d0c94-predictor.18a8c6c3bfef5cb6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: d242fda5-9561-4b96-b173-481122975ab8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:09:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: e4c44752-fede-40dd-8615-c0b81ab3ab9d kind: Event lastTimestamp: "2026-04-22T20:10:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:12Z" name: isvc-logger-raw-d0c94-predictor.18a8c6c3c04faf5b namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: b9fa9777-ef4e-4844-b69e-828b43e24ce7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T20:10:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: e4c44752-fede-40dd-8615-c0b81ab3ab9d kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:57Z" name: isvc-logger-raw-d0c94-predictor.18a8c6cabe12bd22 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 57208a33-06f8-4298-810a-6d52ffcf7324 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T20:10:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: e4c44752-fede-40dd-8615-c0b81ab3ab9d kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:57Z" name: isvc-logger-raw-d0c94-predictor.18a8c6cabe5797bf namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 6adeb55e-37a5-4747-a89b-b94bfaa547de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 17b3f835-2b4f-4762-8568-715f1ad9032a kind: Event lastTimestamp: "2026-04-22T20:10:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:10:49Z" name: isvc-logger-raw-d0c94.18a8c6c040dd60fc namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: ae171524-eec0-4679-b358-672f64693e51 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 17b3f835-2b4f-4762-8568-715f1ad9032a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-d0c94": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-d0c94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94.18a8c6c04168a76a namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 39682ab1-a066-4c8e-91ad-4b5804a79b80 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 17b3f835-2b4f-4762-8568-715f1ad9032a kind: Event lastTimestamp: "2026-04-22T20:09:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-d0c94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:09:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:42Z" name: isvc-logger-raw-d0c94.18a8c6c04594969e namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 9b3b6f71-fcbd-47ef-acda-6ef2f2638047 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 17b3f835-2b4f-4762-8568-715f1ad9032a kind: Event lastTimestamp: "2026-04-22T20:10:49Z" message: InferenceService [isvc-logger-raw-d0c94] is Ready metadata: creationTimestamp: "2026-04-22T20:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:10:49Z" name: isvc-logger-raw-d0c94.18a8c6cff3f446dc namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f020d1ad-10cf-48c6-9c97-8b57a2c75121 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:13:17.571297Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-13f916-predictor-7f4b544bb-7sn97 to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f2709be0e9 namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: 76d1bee6-1310-4268-a391-0d5c78abd5ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17457" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f28920b97d namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: a5824ef4-e9ed-44c1-a3bf-330929a5346f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:18Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f28a481323 namespace: kserve-ci-e2e-test resourceVersion: "17468" uid: 0c17abc8-0b77-4925-9154-efa60baeaf97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:18Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f28d25f529 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: 48970693-5f64-4dfe-9adc-1535f9cf6243 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:18Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f28d92401f namespace: kserve-ci-e2e-test resourceVersion: "17472" uid: 56093361-a110-4c10-8789-4a7ffb35d8a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:21Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f3731cc8da namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 7661a6a4-839b-42ac-830f-878b7f2b92a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:21Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f376481a9d namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: ccd0a36b-4495-48b1-9e2f-83cc52af46ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:13:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:21Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f376d331fa namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: dfa07c15-6f1e-4db5-a725-6bc935816b1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T20:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:43Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c6f3af00ce7b namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: 5aa7f6d5-3f2d-4caa-933e-6e3e064c81bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-13f916-predictor-7f4b544bb-7sn97 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 7c279315-6c3a-4cdd-89fb-e644689ecff5 kind: Event lastTimestamp: "2026-04-22T20:14:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:43Z" name: isvc-primary-13f916-predictor-7f4b544bb-7sn97.18a8c70685593187 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: b6227b26-c072-4516-8160-92fdc49da81d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-13f916-predictor-7f4b544bb namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: cdc42150-2e56-427b-9275-384a3a5d3b35 kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: 'Created pod: isvc-primary-13f916-predictor-7f4b544bb-7sn97' metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916-predictor-7f4b544bb.18a8c6f27045e32a namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 4f1cdfc9-e83a-4e0d-ba23-35971ec7c1cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: cd04449c-aeb0-415c-9851-0f894a4e2c55 kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: Scaled up replica set isvc-primary-13f916-predictor-7f4b544bb from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916-predictor.18a8c6f26bd45311 namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: 31d1b5a9-ab2e-4df6-a136-c2c5b3b6cea4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:13:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "17440" uid: de01432e-9606-4be5-a079-ea6cc8efc6f7 kind: Event lastTimestamp: "2026-04-22T20:13:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:47Z" name: isvc-primary-13f916-predictor.18a8c6f5eae3f6e5 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: b77e7ca2-246d-433e-bce1-8d02462efefb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:13:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "17440" uid: de01432e-9606-4be5-a079-ea6cc8efc6f7 kind: Event lastTimestamp: "2026-04-22T20:13:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:47Z" name: isvc-primary-13f916-predictor.18a8c6f5eb4d895f namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: 1207eeef-0f1c-41b1-ac54-515c7eac43b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T20:14:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: de01432e-9606-4be5-a079-ea6cc8efc6f7 kind: Event lastTimestamp: "2026-04-22T20:14:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:32Z" name: isvc-primary-13f916-predictor.18a8c6fce92bd89e namespace: kserve-ci-e2e-test resourceVersion: "18079" uid: afce5241-7bc6-4255-bd55-a28c1313d252 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T20:14:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: de01432e-9606-4be5-a079-ea6cc8efc6f7 kind: Event lastTimestamp: "2026-04-22T20:14:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:14:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:32Z" name: isvc-primary-13f916-predictor.18a8c6fce98d4372 namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 84e2b52b-ae10-4519-aae4-a88944caef4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 09b12c31-2470-4029-a1bd-67b7d3111d1f kind: Event lastTimestamp: "2026-04-22T20:14:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:24Z" name: isvc-primary-13f916.18a8c6f26bea6339 namespace: kserve-ci-e2e-test resourceVersion: "17958" uid: 706d2d34-1b5b-4827-92db-21dee110001d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 09b12c31-2470-4029-a1bd-67b7d3111d1f kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: 'Failed to update status for InferenceService "isvc-primary-13f916": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-13f916": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916.18a8c6f26c701230 namespace: kserve-ci-e2e-test resourceVersion: "17444" uid: f7c8bb4f-c770-461a-b221-397adf6bd3eb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 09b12c31-2470-4029-a1bd-67b7d3111d1f kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-13f916": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:13:17Z" name: isvc-primary-13f916.18a8c6f26c70596e namespace: kserve-ci-e2e-test resourceVersion: "17446" uid: da932fdc-de8a-4549-a9a6-1ce1455709ef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 09b12c31-2470-4029-a1bd-67b7d3111d1f kind: Event lastTimestamp: "2026-04-22T20:14:23Z" message: InferenceService [isvc-primary-13f916] is Ready metadata: creationTimestamp: "2026-04-22T20:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:23Z" name: isvc-primary-13f916.18a8c701e6d0df04 namespace: kserve-ci-e2e-test resourceVersion: "17957" uid: 334bc17c-ab05-43df-ab88-66b7b4fb1001 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:04:46.393530Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67b6c0a16b7 namespace: kserve-ci-e2e-test resourceVersion: "12951" uid: 48c52e30-c056-400a-9a89-c1e6968fdf36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12956" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67b8513244d namespace: kserve-ci-e2e-test resourceVersion: "12984" uid: 488a6488-7ba9-43bf-8f50-964c64dba123 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67b8677f8b3 namespace: kserve-ci-e2e-test resourceVersion: "12987" uid: d4e16800-d8f8-4612-bfd1-f79ab7ce8a55 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:51Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" in 4.476s (4.476s including waiting). Image size: 300957093 bytes.' metadata: creationTimestamp: "2026-04-22T20:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:51Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67c9143c5bf namespace: kserve-ci-e2e-test resourceVersion: "13019" uid: c3aa99e1-692c-4502-9d9f-ae0c6d22874d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:51Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67c93eb89c1 namespace: kserve-ci-e2e-test resourceVersion: "13020" uid: 56076403-f8bc-4254-9d65-ced74ea6f912 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:51Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67c9451f181 namespace: kserve-ci-e2e-test resourceVersion: "13021" uid: 49c72c4f-bff0-4c87-8669-4690468aefc9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:04:55Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" metadata: creationTimestamp: "2026-04-22T20:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:04:55Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c67d8c5ed129 namespace: kserve-ci-e2e-test resourceVersion: "13057" uid: a8b89b76-2e19-4b77-af22-35f7e25f6376 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" in 14.076s (14.076s including waiting). Image size: 1562514860 bytes.' metadata: creationTimestamp: "2026-04-22T20:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:09Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c680d35f3b70 namespace: kserve-ci-e2e-test resourceVersion: "13160" uid: e78d1dd5-9c2f-4ee9-89ca-547dc19bf2df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:09Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c680d62ea70e namespace: kserve-ci-e2e-test resourceVersion: "13162" uid: 68bee802-48d1-4228-937f-c766bc653138 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:09Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c680d6992245 namespace: kserve-ci-e2e-test resourceVersion: "13163" uid: 9f79e252-013d-48e5-a3d9-1ea4605d27ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:09Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:84e96718cb727bdb7a70c88a1b15da870dac18fe5721015f96b729224acb0c47" metadata: creationTimestamp: "2026-04-22T20:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:09Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c680d6a3b6f8 namespace: kserve-ci-e2e-test resourceVersion: "13164" uid: 5c47a6d2-ad30-48ad-92a0-742cd5c156ae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:12Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:84e96718cb727bdb7a70c88a1b15da870dac18fe5721015f96b729224acb0c47" in 2.638s (2.638s including waiting). Image size: 237896263 bytes.' metadata: creationTimestamp: "2026-04-22T20:05:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:12Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c68173e920bb namespace: kserve-ci-e2e-test resourceVersion: "13183" uid: 671ad57b-2817-43d1-bfbc-3bc12669acd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:12Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T20:05:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:12Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c681774b85f5 namespace: kserve-ci-e2e-test resourceVersion: "13184" uid: 581dd3d5-2f97-43ed-9c6a-5b25a5c5b96f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:05:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:05:12Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T20:05:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:05:12Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c68177bb4037 namespace: kserve-ci-e2e-test resourceVersion: "13185" uid: 62e6a2a8-8d2a-4638-8d09-31646cbc09bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T20:05:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:06:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:05:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:44Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c68184e2601c namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: c30579ba-a348-4e06-a924-535fba53937b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:06:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:44Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c681c097ae1d namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: 9f1ad947-f6be-4fc2-ac09-e2d07843ebec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:21Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c6918f658a60 namespace: kserve-ci-e2e-test resourceVersion: "13723" uid: 9eb7a7a3-ae15-4752-9e85-a84986158b01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 6c7431a5-1545-45be-9b51-ac8361ee3fcc kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:21Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m.18a8c6918f668ab6 namespace: kserve-ci-e2e-test resourceVersion: "13724" uid: e527b706-e6d8-4678-9d4c-2eda62a837f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf namespace: kserve-ci-e2e-test resourceVersion: "12932" uid: c9892cf3-64eb-46ee-b64c-1850559eda28 kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: 'Created pod: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf-j6t2m' metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor-55776767cf.18a8c67b6b9efc5d namespace: kserve-ci-e2e-test resourceVersion: "12947" uid: cfcc92c0-2787-404c-b1fa-aa3a0a9a4209 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-586f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 1f3e9ec7-55a7-4242-8482-ccd656653091 kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: Scaled up replica set isvc-raw-sklearn-batcher-586f1-predictor-55776767cf from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor.18a8c67b63d1d8ed namespace: kserve-ci-e2e-test resourceVersion: "12933" uid: d711fd94-e0db-47d2-a2e9-1266a6062a61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-586f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12941" uid: 6ed99216-df0b-4f50-935b-9d02fcecaaa0 kind: Event lastTimestamp: "2026-04-22T20:05:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:05:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor.18a8c67ee92bc72c namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: 10254081-1794-4604-9a04-62c11356e78c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-586f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12941" uid: 6ed99216-df0b-4f50-935b-9d02fcecaaa0 kind: Event lastTimestamp: "2026-04-22T20:05:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:05:46Z" name: isvc-raw-sklearn-batcher-586f1-predictor.18a8c67ee989059b namespace: kserve-ci-e2e-test resourceVersion: "13433" uid: 7dfee60a-efeb-4f41-b4e9-1ecc898f5b3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:06:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-586f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13099" uid: 6ed99216-df0b-4f50-935b-9d02fcecaaa0 kind: Event lastTimestamp: "2026-04-22T20:06:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:16Z" name: isvc-raw-sklearn-batcher-586f1-predictor.18a8c68ce6179608 namespace: kserve-ci-e2e-test resourceVersion: "13649" uid: fac8caed-00b3-412b-8098-b3d502f45774 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:06:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-586f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13099" uid: 6ed99216-df0b-4f50-935b-9d02fcecaaa0 kind: Event lastTimestamp: "2026-04-22T20:06:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:16Z" name: isvc-raw-sklearn-batcher-586f1-predictor.18a8c68ce688af0d namespace: kserve-ci-e2e-test resourceVersion: "13650" uid: f9ce0fa0-9587-4244-aef8-eaaa3feac087 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-586f1 namespace: kserve-ci-e2e-test resourceVersion: "12923" uid: 751f6480-3d79-4482-983c-9d172f48915d kind: Event lastTimestamp: "2026-04-22T20:06:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:06:14Z" name: isvc-raw-sklearn-batcher-586f1.18a8c67b69dfa0e5 namespace: kserve-ci-e2e-test resourceVersion: "13637" uid: 29da3c13-65f7-4ac9-832d-498666153667 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-586f1 namespace: kserve-ci-e2e-test resourceVersion: "12923" uid: 751f6480-3d79-4482-983c-9d172f48915d kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-586f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-586f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1.18a8c67b6a4e7ad6 namespace: kserve-ci-e2e-test resourceVersion: "12943" uid: 08e8d207-5746-4fa7-9a6a-7cb4590e69a2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:04:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-586f1 namespace: kserve-ci-e2e-test resourceVersion: "12923" uid: 751f6480-3d79-4482-983c-9d172f48915d kind: Event lastTimestamp: "2026-04-22T20:04:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-586f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:04:46Z" name: isvc-raw-sklearn-batcher-586f1.18a8c67b6a4ee211 namespace: kserve-ci-e2e-test resourceVersion: "12945" uid: 8cf77fb4-cfe9-4b77-93cf-3f7a778abe7a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-586f1 namespace: kserve-ci-e2e-test resourceVersion: "13635" uid: 751f6480-3d79-4482-983c-9d172f48915d kind: Event lastTimestamp: "2026-04-22T20:06:14Z" message: InferenceService [isvc-raw-sklearn-batcher-586f1] is Ready metadata: creationTimestamp: "2026-04-22T20:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:06:14Z" name: isvc-raw-sklearn-batcher-586f1.18a8c68ffa1594d0 namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: e9e99d11-a4b7-4142-b7da-6503f59e36ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:14:27.691896Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18015" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-13f916-predictor-d9cf89fc5-skn6q to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:14:27Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c702c41f2f33 namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: e2a03079-a7b5-4c80-817c-a809e06b7702 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: "2026-04-22T20:14:28Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:14:28Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c702dcdd3e94 namespace: kserve-ci-e2e-test resourceVersion: "18029" uid: cd708b79-75f6-446e-a110-8c1428d27c0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: "2026-04-22T20:14:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:32Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c702de13d623 namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: fde9fd68-0525-4f9d-8cda-679424150b1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: "2026-04-22T20:14:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:32Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c702e0dff5bf namespace: kserve-ci-e2e-test resourceVersion: "18072" uid: 60b511be-a3d0-4312-8d55-d27f0411fa07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:14:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: "2026-04-22T20:14:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:32Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c702e14d5986 namespace: kserve-ci-e2e-test resourceVersion: "18075" uid: 62f9e843-0b0c-4094-a97a-9e758b63707a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: e98535e9-0e7b-41a9-b598-b082c4c7d137 kind: Event lastTimestamp: "2026-04-22T20:14:38Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-13f916-predictor-d9cf89fc5-skn6q_kserve-ci-e2e-test(e98535e9-0e7b-41a9-b598-b082c4c7d137) metadata: creationTimestamp: "2026-04-22T20:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:38Z" name: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q.18a8c705319e0578 namespace: kserve-ci-e2e-test resourceVersion: "18121" uid: 25a71560-32aa-4f3e-9a5a-d7bb03f0bf93 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-13f916-predictor-d9cf89fc5 namespace: kserve-ci-e2e-test resourceVersion: "18004" uid: afed153f-24a8-4073-8ac8-f09f5affc44c kind: Event lastTimestamp: "2026-04-22T20:14:27Z" message: 'Created pod: isvc-secondary-13f916-predictor-d9cf89fc5-skn6q' metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:27Z" name: isvc-secondary-13f916-predictor-d9cf89fc5.18a8c702c3baa6ab namespace: kserve-ci-e2e-test resourceVersion: "18016" uid: d0251e73-82d1-4b38-a686-f72747ad357f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "18001" uid: b1cb467a-77fe-4186-83b6-a393378d7bd9 kind: Event lastTimestamp: "2026-04-22T20:14:27Z" message: Scaled up replica set isvc-secondary-13f916-predictor-d9cf89fc5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:27Z" name: isvc-secondary-13f916-predictor.18a8c702c0847ac5 namespace: kserve-ci-e2e-test resourceVersion: "18005" uid: efc0c3a4-3258-4096-aec2-6734c5b21e64 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "18003" uid: 9e848b1f-2f16-4435-8764-0141fe489b39 kind: Event lastTimestamp: "2026-04-22T20:14:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:14:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:42Z" name: isvc-secondary-13f916-predictor.18a8c7063f3a69be namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 622f47f4-0259-4d50-8280-ee86a106ba75 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-13f916-predictor namespace: kserve-ci-e2e-test resourceVersion: "18003" uid: 9e848b1f-2f16-4435-8764-0141fe489b39 kind: Event lastTimestamp: "2026-04-22T20:14:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:14:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:42Z" name: isvc-secondary-13f916-predictor.18a8c7063fa37292 namespace: kserve-ci-e2e-test resourceVersion: "18162" uid: e83a4cb8-0a9c-41e3-842e-516ab0d78d37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T20:14:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17993" uid: 8b9619a4-82de-40ff-bcf4-20820900255b kind: Event lastTimestamp: "2026-04-22T20:14:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:38Z" name: isvc-secondary-13f916.18a8c702c088f53c namespace: kserve-ci-e2e-test resourceVersion: "18130" uid: b2db3e63-06e7-4076-b7c6-95aa9601ca93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17993" uid: 8b9619a4-82de-40ff-bcf4-20820900255b kind: Event lastTimestamp: "2026-04-22T20:14:27Z" message: 'Failed to update status for InferenceService "isvc-secondary-13f916": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-13f916": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:27Z" name: isvc-secondary-13f916.18a8c702c1131d68 namespace: kserve-ci-e2e-test resourceVersion: "18008" uid: 7f57f83a-1150-403a-98e0-cee6e3a474fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-13f916 namespace: kserve-ci-e2e-test resourceVersion: "17993" uid: 8b9619a4-82de-40ff-bcf4-20820900255b kind: Event lastTimestamp: "2026-04-22T20:14:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-13f916": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:27Z" name: isvc-secondary-13f916.18a8c702c11375a9 namespace: kserve-ci-e2e-test resourceVersion: "18010" uid: f9bee3b0-b4d9-4a2d-9e7f-70cea7b2b1d3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:06:21.537135Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13733" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c691930a48c8 namespace: kserve-ci-e2e-test resourceVersion: "13747" uid: d0733cff-5a82-4065-a9d3-850fabfb149d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c691ac25564e namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: 4564afc3-e210-4350-b066-935b5e60fe08 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c691ad524b72 namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: 872af467-08d2-479b-b355-6fd6de5c2cb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:22Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c691b18138f9 namespace: kserve-ci-e2e-test resourceVersion: "13787" uid: 89fa72f6-d6a3-4c09-a6ad-e4092a5d1bac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:22Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c691b1f73b95 namespace: kserve-ci-e2e-test resourceVersion: "13788" uid: 96182fe9-2ded-4924-abec-96fb7b36b7c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:25Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c6928fe76ddb namespace: kserve-ci-e2e-test resourceVersion: "13836" uid: 2bdbc077-fbcf-492a-85a5-197e5cc8032f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:25Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c69292aa08e6 namespace: kserve-ci-e2e-test resourceVersion: "13840" uid: d72bdb37-f171-49df-a73b-223b669ac4f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:06:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:25Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c692931704ca namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 91527185-e967-4ae1-ad83-acdef25e64ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T20:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:07:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:27Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c692cbe00cb1 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: dd0b1d3b-32d0-434b-968a-621b30572ade reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: c28960d2-0a72-41e2-b831-f4e123784e15 kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:01Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m.18a8c6a8e906ce9e namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: d593dfc6-4fc3-46f9-b226-2979d7622804 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5 namespace: kserve-ci-e2e-test resourceVersion: "13715" uid: 033667c5-9b03-4ce2-8585-2e95845d435b kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: 'Created pod: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5-jj65m' metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5.18a8c691928dbf4b namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: 25103130-377b-4442-a1cd-8bc17f3413cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13710" uid: cc6c1d5b-049b-4b1c-b4e3-d2d8b1055c0e kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Scaled up replica set isvc-sklearn-graph-raw-81bcf-predictor-745f867bd5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf-predictor.18a8c6918e168e20 namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: dd16e13e-3e55-4ae4-9616-06fad71276eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 88622aa4-ece9-4962-9fbb-f6f4e7f71337 kind: Event lastTimestamp: "2026-04-22T20:06:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:36Z" name: isvc-sklearn-graph-raw-81bcf-predictor.18a8c6950cc35ad3 namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: da6777ba-1387-40b4-9362-461a58a6c72e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 88622aa4-ece9-4962-9fbb-f6f4e7f71337 kind: Event lastTimestamp: "2026-04-22T20:06:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:36Z" name: isvc-sklearn-graph-raw-81bcf-predictor.18a8c6950d231fad namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: fb4d166b-7c1d-4fcc-a68f-7eb659a912eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T20:06:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: 88622aa4-ece9-4962-9fbb-f6f4e7f71337 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: isvc-sklearn-graph-raw-81bcf-predictor.18a8c6988c2f6571 namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: a2613841-bd65-4a56-94e4-2e1d58806184 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T20:06:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: 88622aa4-ece9-4962-9fbb-f6f4e7f71337 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: isvc-sklearn-graph-raw-81bcf-predictor.18a8c6988ca1aedb namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: c0a987ca-8f7c-40b3-8f8a-c39be3a4ad9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "13691" uid: cf7c9a84-50f3-4ba3-a6f9-04779436595c kind: Event lastTimestamp: "2026-04-22T20:07:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:07:37Z" name: isvc-sklearn-graph-raw-81bcf.18a8c6918e20468f namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: dc41f954-e10c-49e4-a9b4-08c9b5b86971 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "13691" uid: cf7c9a84-50f3-4ba3-a6f9-04779436595c kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-81bcf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-81bcf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-sklearn-graph-raw-81bcf.18a8c6918e8bb8e1 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 7ccf076f-4dd9-4572-ad49-7a815feb659e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: cf7c9a84-50f3-4ba3-a6f9-04779436595c kind: Event lastTimestamp: "2026-04-22T20:07:37Z" message: InferenceService [isvc-sklearn-graph-raw-81bcf] is Ready metadata: creationTimestamp: "2026-04-22T20:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:07:37Z" name: isvc-sklearn-graph-raw-81bcf.18a8c6a35817cfae namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: 575b5f11-17b6-4a5d-a94d-6e2d3f7c91ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:08:01.792188Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:08:01Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6a8eab51520 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: a59c8927-550b-4253-8683-98cec19df57d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:08:02Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6a9051f5dac namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 857fff80-7065-415d-900d-db2e2142edd1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6a9065db596 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: c044e1b1-0865-4e57-b95a-688fba644512 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6a909b013c3 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 83a0d73d-b3e9-4b44-9ff3-68b7e2ae9910 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6a90a283a08 namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: ac0ffe05-2444-4f97-9dcf-4c0834301dd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6aa25cfd66d namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 4cf0a039-fd4f-4065-b29f-eeb1506612aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6aa28f1617d namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: df5d353d-12d2-42af-a8a8-301f53726264 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6aa29601c35 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: f832d162-d5f4-466d-991d-b622f7a2cbdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T20:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:08:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:59Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6aa61fbe788 namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 2267eaaf-a5a6-422e-ba63-60e748b1eadb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: ebf23c2a-33ad-46fc-9385-3cbe6370d937 kind: Event lastTimestamp: "2026-04-22T20:09:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:32Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr.18a8c6bdedd9ab6d namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: f2d0c44f-fbd7-45ce-a7ce-c129fb0ebeb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: fee1b4d7-2621-4a2c-ba2c-c78d643dbfeb kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889-x96mr' metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:01Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889.18a8c6a8ea337542 namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: dc792897-34ec-4c50-8baf-be14d14a15b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 07890fdb-8c1e-43a2-8172-a9251ca5d025 kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3e39c-predictor-57c8568889 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:01Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor.18a8c6a8e666dda1 namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: 4cae467f-8564-4701-8237-9a6e37358c46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 01631cf6-1cd5-4815-8bce-0b99aa02615c kind: Event lastTimestamp: "2026-04-22T20:08:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:16Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor.18a8c6ac654adb9e namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: 45b42190-f20e-49c4-a214-d7bd9cd20a14 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 01631cf6-1cd5-4815-8bce-0b99aa02615c kind: Event lastTimestamp: "2026-04-22T20:08:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:16Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor.18a8c6ac65a359d1 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: f25dc92d-1bc1-46b3-b519-31b31383e684 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T20:08:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 01631cf6-1cd5-4815-8bce-0b99aa02615c kind: Event lastTimestamp: "2026-04-22T20:09:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:31Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor.18a8c6afe540aaed namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: e64e6481-9aef-4872-a0b9-567169341403 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T20:08:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 01631cf6-1cd5-4815-8bce-0b99aa02615c kind: Event lastTimestamp: "2026-04-22T20:09:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:31Z" name: isvc-sklearn-graph-raw-hpa-3e39c-predictor.18a8c6afe5a7078a namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: ae774a97-66cd-4057-9ca0-dbf64af6ed97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 9df31d5d-be7f-4efb-ade7-e9748d782420 kind: Event lastTimestamp: "2026-04-22T20:09:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:09Z" name: isvc-sklearn-graph-raw-hpa-3e39c.18a8c6a8e69f1cc2 namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: d8812e3d-24db-4b45-98ab-828d859e3bfe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 9df31d5d-be7f-4efb-ade7-e9748d782420 kind: Event lastTimestamp: "2026-04-22T20:09:09Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3e39c] is Ready metadata: creationTimestamp: "2026-04-22T20:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:09Z" name: isvc-sklearn-graph-raw-hpa-3e39c.18a8c6b89a94238e namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0cbe8e51-3ef2-49f8-9d1b-c6fdde28c3c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:10:57.292025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d1c74ff867 namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: 573bbd94-3497-4292-835b-e482bad5c6ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d1e0076cc5 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: afcfbd76-5e64-4313-8e8f-cbab41391a4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d1e13aa8c3 namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: 413795ea-65cb-48eb-a5d7-9c321635870b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d1e40138d7 namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: b89c2f65-3078-4b01-a007-f00d7fa96de6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d1e46e70ec namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 6dbcf052-5f90-49e1-b09b-0263774ef7ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:11:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:01Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d2c53aadf2 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 7b2f5c47-3141-4446-905d-4ed51b9161e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:11:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:01Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d2c96b45a7 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: ba0c4b39-f3ac-4b81-9428-ce4a7fc09ec7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:11:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:11:01Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d2c9d84d8f namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: fabcebb4-3481-48a0-9240-71c09c04acfa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-04-22T20:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:13:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:20Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6d30149c3c7 namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: 08b4fdc7-47de-40f8-a386-3ee69507aba2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: c85295af-4265-48b5-9a4c-ac42f718f1b1 kind: Event lastTimestamp: "2026-04-22T20:13:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:13:17Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j.18a8c6f26aa95335 namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: 97f8b160-ddaf-4fbd-a338-caf12611fa44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 3c0720f3-ea7d-421b-8ae9-4fce4e318367 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: 'Created pod: isvc-sklearn-scale-raw-45df6-predictor-854b479db7-7rn5j' metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor-854b479db7.18a8c6d1c6d2a7a5 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 49485490-52d0-4e63-ba6e-1256ef5e99a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-45df6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: d4fc9b90-b3bf-4532-81d4-1891cde30f73 kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Scaled up replica set isvc-sklearn-scale-raw-45df6-predictor-854b479db7 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6-predictor.18a8c6d1c499e7d6 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: b2462bee-2408-4d4f-bde7-5ef0f8786124 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-45df6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 80334e05-1114-4c4d-aeab-c4b4d84c62c9 kind: Event lastTimestamp: "2026-04-22T20:11:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:11:12Z" name: isvc-sklearn-scale-raw-45df6-predictor.18a8c6d5434f81da namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: cfe9b58d-a9d3-47d0-a7d4-35bc57f87d3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:11:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-45df6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 80334e05-1114-4c4d-aeab-c4b4d84c62c9 kind: Event lastTimestamp: "2026-04-22T20:11:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:11:12Z" name: isvc-sklearn-scale-raw-45df6-predictor.18a8c6d543b0496a namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 49d44f42-e70e-46fe-be24-e4cf1ce67646 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T20:11:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-45df6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 80334e05-1114-4c4d-aeab-c4b4d84c62c9 kind: Event lastTimestamp: "2026-04-22T20:13:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:12Z" name: isvc-sklearn-scale-raw-45df6-predictor.18a8c6d8c2cb45d9 namespace: kserve-ci-e2e-test resourceVersion: "17368" uid: cb163bab-e84c-4850-ad39-f42f315069fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T20:11:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-45df6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 80334e05-1114-4c4d-aeab-c4b4d84c62c9 kind: Event lastTimestamp: "2026-04-22T20:13:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:13:12Z" name: isvc-sklearn-scale-raw-45df6-predictor.18a8c6d8c329fe79 namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: 39c5476f-d80b-4751-827d-925794643e09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-45df6 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: ad38acb2-5cce-4c60-8bff-48a11e3de08b kind: Event lastTimestamp: "2026-04-22T20:13:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:13:10Z" name: isvc-sklearn-scale-raw-45df6.18a8c6d1c4a12b5d namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: f9e6d15c-8b7a-4ccc-947e-b1e95691d5e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-45df6 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: ad38acb2-5cce-4c60-8bff-48a11e3de08b kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-45df6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-45df6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:10:57Z" name: isvc-sklearn-scale-raw-45df6.18a8c6d1c51402a3 namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: f462dfd7-d050-45bd-86fc-119a4a742933 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:13:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-45df6 namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: ad38acb2-5cce-4c60-8bff-48a11e3de08b kind: Event lastTimestamp: "2026-04-22T20:13:10Z" message: InferenceService [isvc-sklearn-scale-raw-45df6] is Ready metadata: creationTimestamp: "2026-04-22T20:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:13:10Z" name: isvc-sklearn-scale-raw-45df6.18a8c6f0c47f275e namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: 1601373a-4f25-4468-b3d3-8dcf2a2cf86d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:06:21.586156Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:06:21Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c69195f63e57 namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 4bfbc85d-3811-4ece-9dcf-0d4ce0abdd4c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:06:22Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c691af796d4c namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: cd9f02bf-a8d5-40d6-a513-be8d422b0a77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:22Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c691b11afb33 namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 705a8e3d-c230-4242-a067-30e07aed367d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:22Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c691b37f5d99 namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: 9aa7a977-efd5-483c-a55b-cd7fe07366fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:22Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c691b406571e namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: f57395bc-a08f-4cd1-942f-d15140029dca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:26Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T20:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:26Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c692cbc74cc1 namespace: kserve-ci-e2e-test resourceVersion: "13848" uid: 84330b71-5049-4ab5-8a10-13df82572fe1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:47Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.208s (21.208s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T20:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:47Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c697bbe900a7 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 2689a599-bbdb-4f90-833d-c200b480f9cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:48Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c697bf294f46 namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 74ddf946-d178-47ef-b63f-201bdfe675c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:06:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:06:48Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c697bfac36b8 namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: 3d2293a4-29fd-4da7-95ae-40196d9f3f9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T20:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:07:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:39Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c697ef869fa1 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: d50f16c6-e797-4594-b8b3-8ea30273066a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 5ad94efb-b19f-4f67-8525-442359a14999 kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:01Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx.18a8c6a8f4de3fa8 namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 530c4f52-96af-4e9f-82ec-10ea3d3e8306 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf namespace: kserve-ci-e2e-test resourceVersion: "13735" uid: 023b188b-190e-4de1-97e6-c382f78197e9 kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: 'Created pod: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf-7nlsx' metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf.18a8c691957d0efe namespace: kserve-ci-e2e-test resourceVersion: "13758" uid: 067ad4c0-7aac-48e1-89c8-f006bb9cb49a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: 84e0eb87-daa2-4575-835d-e29596b04b12 kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: Scaled up replica set isvc-xgboost-graph-raw-81bcf-predictor-65c8ddb5cf from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-xgboost-graph-raw-81bcf-predictor.18a8c69192d49975 namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 5f5280ec-8bf4-42bd-b68a-2967a90bc8e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 0f02bb71-c50d-443a-a875-14dec2b391e9 kind: Event lastTimestamp: "2026-04-22T20:07:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:21Z" name: isvc-xgboost-graph-raw-81bcf-predictor.18a8c69511eb83fb namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: b64afab5-7211-40c6-9684-968462276be3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: 0f02bb71-c50d-443a-a875-14dec2b391e9 kind: Event lastTimestamp: "2026-04-22T20:07:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:21Z" name: isvc-xgboost-graph-raw-81bcf-predictor.18a8c69512e3f821 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: dfb34cc8-8b2f-4bf5-a3c4-41d94ef33591 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: 0f02bb71-c50d-443a-a875-14dec2b391e9 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: isvc-xgboost-graph-raw-81bcf-predictor.18a8c6a30da40b56 namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: e1edf904-21bf-4716-89b2-ba25f8ea9b29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-81bcf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: 0f02bb71-c50d-443a-a875-14dec2b391e9 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: isvc-xgboost-graph-raw-81bcf-predictor.18a8c6a30e063602 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 1a0e6b64-f3b5-4c57-b784-6695f9ea2699 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "13725" uid: e7d8daa7-1bd8-41d3-8aa2-28185e71d3f6 kind: Event lastTimestamp: "2026-04-22T20:07:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:07:49Z" name: isvc-xgboost-graph-raw-81bcf.18a8c69192a7c3ed namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 8c29fa3c-0a96-426e-8289-be0521fda02f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "13725" uid: e7d8daa7-1bd8-41d3-8aa2-28185e71d3f6 kind: Event lastTimestamp: "2026-04-22T20:06:21Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-81bcf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-81bcf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:06:21Z" name: isvc-xgboost-graph-raw-81bcf.18a8c691934439a5 namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: c1c666d6-b039-400d-b375-d53784ea39ee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: e7d8daa7-1bd8-41d3-8aa2-28185e71d3f6 kind: Event lastTimestamp: "2026-04-22T20:07:49Z" message: InferenceService [isvc-xgboost-graph-raw-81bcf] is Ready metadata: creationTimestamp: "2026-04-22T20:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:07:49Z" name: isvc-xgboost-graph-raw-81bcf.18a8c6a6277588ed namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 85fb94ff-89f9-4334-a8c2-0986c2d615e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:08:01.864543Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:08:01Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6a8ef050ea8 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: fdc72c14-3132-4366-b441-4a12af6add57 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:08:02Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6a908bcce28 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: eb05020f-6e62-4755-aeda-a30c4184441c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6a909f594a0 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 1843ab0d-e3f9-4942-ad3f-db001d05017d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6a90cb4403c namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: d6911224-969e-4256-add9-2f63e557baed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:02Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6a90d232b07 namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: fa08671e-ac66-48b3-9cae-7e654802547f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6aa25ac3c77 namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: ac59c98d-9148-41bb-ac25-2cf07fe6ab82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6aa28df7681 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: a5298537-1b09-4162-9eb6-715cff12bcef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:07Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6aa294f37d1 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 0c67de43-892d-47aa-962b-cec8c0d1d24c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T20:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:08:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:59Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6aa620a2034 namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 1973b5ae-1948-4645-a619-d2edd5cfd54c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 296784ea-f704-464e-be95-0525b956e003 kind: Event lastTimestamp: "2026-04-22T20:09:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:31Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt.18a8c6bdeb1e6b9c namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 373949c0-a551-4a14-a007-fa52cfed6b52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: ac2c9a87-d95f-41c9-b45f-7a80425e72ed kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947-6rttt' metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:01Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947.18a8c6a8ee9be7e7 namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 4ab051a0-5032-4822-ae19-ad56c528599d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 6914e9e2-458b-4b2a-b4d6-0e4c23e9208a kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3e39c-predictor-7cf7df947 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:01Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor.18a8c6a8ebfee186 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 5ae381dc-d139-4ba2-b5a2-59a4f4d7cccb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 0f0bd88a-b1a2-4471-b7d7-ecd17b2f8816 kind: Event lastTimestamp: "2026-04-22T20:08:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:16Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor.18a8c6ac6a6f8d6f namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: a3f719b4-a8ce-43b8-b5a2-8c8a844da7e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 0f0bd88a-b1a2-4471-b7d7-ecd17b2f8816 kind: Event lastTimestamp: "2026-04-22T20:08:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:08:16Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor.18a8c6ac6aeab7e5 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 6dbef1ec-1aab-48ea-a295-d45d65756a29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:08:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 0f0bd88a-b1a2-4471-b7d7-ecd17b2f8816 kind: Event lastTimestamp: "2026-04-22T20:09:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:16Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor.18a8c6afe9bc1d2c namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 358f307c-c956-4a58-8979-75bf0ff28c19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T20:08:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e39c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 0f0bd88a-b1a2-4471-b7d7-ecd17b2f8816 kind: Event lastTimestamp: "2026-04-22T20:09:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:16Z" name: isvc-xgboost-graph-raw-hpa-3e39c-predictor.18a8c6afea20f054 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 96b865d4-9063-4cbb-abc5-51fc5e02d068 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: e463803b-843e-4782-b719-5c2e322946cc kind: Event lastTimestamp: "2026-04-22T20:09:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:09Z" name: isvc-xgboost-graph-raw-hpa-3e39c.18a8c6a8ebcf878d namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: d82e1b84-8b91-44e9-9e3d-44e6f80d0323 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: e463803b-843e-4782-b719-5c2e322946cc kind: Event lastTimestamp: "2026-04-22T20:09:09Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3e39c] is Ready metadata: creationTimestamp: "2026-04-22T20:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:09Z" name: isvc-xgboost-graph-raw-hpa-3e39c.18a8c6b89cec3b0a namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 067004b8-09d4-4c5b-af27-bd9c2b2ff2ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:09:32.049258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 to ip-10-0-135-72.ec2.internal metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:09:32Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6bdee728904 namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: b4916a7a-7893-4d6b-a443-4b914d772959 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:32Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:09:32Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:09:32Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6bdf62848d0 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: bbd9ac55-a4df-4acd-8684-cd45f9f553cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:09:32Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:32Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6bdf752e5e3 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 45c1935a-eb02-4659-9cf3-d954289330e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-72.ec2.internal source: component: kubelet host: ip-10-0-135-72.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:09:33Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 967ms (967ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-22T20:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:33Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6be30fd2df7 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 2fea32b7-d892-40fe-bb6c-9db271d4a600 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-72.ec2.internal source: component: kubelet host: ip-10-0-135-72.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:09:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:33Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6be362e896a namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 57ff612d-d40f-49ae-b89b-43fe5992f01e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-72.ec2.internal source: component: kubelet host: ip-10-0-135-72.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:09:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:33Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6be3695e5f6 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: b1e53875-0910-4f76-b1c0-aea47fff64b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-72.ec2.internal source: component: kubelet host: ip-10-0-135-72.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: e5eee0d5-20fc-4ef7-9383-b6b499dc986b kind: Event lastTimestamp: "2026-04-22T20:10:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:10:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:10:57Z" name: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5.18a8c6d1cb2aae57 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: cddcbd42-7cd4-4396-975a-0cbc17881cc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-72.ec2.internal source: component: kubelet host: ip-10-0-135-72.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-d0c94-predictor-68c744c676 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 53d009e0-25b2-44ee-b8c9-ff3b8a5ac8b1 kind: Event lastTimestamp: "2026-04-22T20:09:32Z" message: 'Created pod: message-dumper-raw-d0c94-predictor-68c744c676-9sbm5' metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:32Z" name: message-dumper-raw-d0c94-predictor-68c744c676.18a8c6bdede49aa5 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 70a4a6b6-89f5-4a1e-9e93-55c56b32b26f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 5df5293c-5362-47b2-9f4e-d26e22eee351 kind: Event lastTimestamp: "2026-04-22T20:09:31Z" message: Scaled up replica set message-dumper-raw-d0c94-predictor-68c744c676 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:31Z" name: message-dumper-raw-d0c94-predictor.18a8c6bdea9993a4 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 27d69f74-778a-48ac-a87b-9ad483e171dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 45ca0b7d-41a2-4978-b0f4-a31be9a09c44 kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:46Z" name: message-dumper-raw-d0c94-predictor.18a8c6c168fb27cb namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: e121d830-8769-4a7f-a799-25f46cfe31a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 45ca0b7d-41a2-4978-b0f4-a31be9a09c44 kind: Event lastTimestamp: "2026-04-22T20:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:46Z" name: message-dumper-raw-d0c94-predictor.18a8c6c169550338 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 4fa72e43-e96e-4b95-a68d-9719808aa5cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 45ca0b7d-41a2-4978-b0f4-a31be9a09c44 kind: Event lastTimestamp: "2026-04-22T20:10:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:02Z" name: message-dumper-raw-d0c94-predictor.18a8c6c4e7ebb061 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: fe50c731-bef7-4021-b87c-82d4ee0e02cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:10:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-d0c94-predictor namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 45ca0b7d-41a2-4978-b0f4-a31be9a09c44 kind: Event lastTimestamp: "2026-04-22T20:10:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:10:02Z" name: message-dumper-raw-d0c94-predictor.18a8c6c4e84c216c namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 55b6cf7c-c93d-4457-bc2b-cde57b9da9b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T20:09:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: b8674835-f6cf-4cc2-9c9f-b0456a057912 kind: Event lastTimestamp: "2026-04-22T20:09:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:33Z" name: message-dumper-raw-d0c94.18a8c6bdea61b226 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 5d9eae2a-f480-485b-892d-e59abd7cee0e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: b8674835-f6cf-4cc2-9c9f-b0456a057912 kind: Event lastTimestamp: "2026-04-22T20:09:32Z" message: 'Failed to update status for InferenceService "message-dumper-raw-d0c94": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-d0c94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:32Z" name: message-dumper-raw-d0c94.18a8c6bded42c349 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 9c8d24fc-d0bd-47d5-a4dc-9488afcb8074 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-d0c94 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: b8674835-f6cf-4cc2-9c9f-b0456a057912 kind: Event lastTimestamp: "2026-04-22T20:09:33Z" message: InferenceService [message-dumper-raw-d0c94] is Ready metadata: creationTimestamp: "2026-04-22T20:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:33Z" name: message-dumper-raw-d0c94.18a8c6be4a58536d namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: c8e850a8-7c1d-4610-9303-47cd9868e83d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:07:51.576731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-81bcf-f88d6597b-xj4kr to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:07:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:07:51Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a689d181a5 namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: bed5f8cf-40f7-4cf4-af6c-cc3cc3305fb8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:51Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-81bcf-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:51Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a69346e12b namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 60d43dd7-9a4d-42c3-9ada-e18702063c95 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:52Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:07:52Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a6c6244391 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 88560b0d-f0cc-4d9c-bc0f-a71514bc8157 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:52Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c077214438a2f9bdb936ff2da330f1b072e635107d34dd2fdb9da07fac4424c5" metadata: creationTimestamp: "2026-04-22T20:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:52Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a6c76e5056 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: 794978cb-5f18-42de-a34f-b75a5f74b49b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:54Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c077214438a2f9bdb936ff2da330f1b072e635107d34dd2fdb9da07fac4424c5" in 2.384s (2.384s including waiting). Image size: 216232519 bytes.' metadata: creationTimestamp: "2026-04-22T20:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:54Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a755950959 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 1a208dc6-66b4-4f6f-acfd-4ddd25f1de6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:55Z" message: 'Created container: model-chainer-raw-81bcf' metadata: creationTimestamp: "2026-04-22T20:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:55Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a7586895b6 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 2cfa149b-504b-4fda-9ea4-bf5ac131446a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:07:55Z" message: Started container model-chainer-raw-81bcf metadata: creationTimestamp: "2026-04-22T20:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:07:55Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a758d4b177 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 0b2428f2-0cd9-400e-8434-63fe04ad404e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:08:01Z" message: Stopping container model-chainer-raw-81bcf metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:01Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a8dfdc9bb7 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: ed04bcf7-f20d-4f03-8182-af2649e5edc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T20:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:08:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:26Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6a8e0472030 namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: bd00fa76-1acd-424d-82ed-0f540153c19f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-81bcf} kind: Pod name: model-chainer-raw-81bcf-f88d6597b-xj4kr namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 864b82cb-96bf-4004-a936-729e7e69e8b8 kind: Event lastTimestamp: "2026-04-22T20:08:31Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8080/readyz": dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:08:31Z" name: model-chainer-raw-81bcf-f88d6597b-xj4kr.18a8c6afdc3e50f2 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 5fd981c7-77c8-4588-86e2-87578336709b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-81bcf-f88d6597b namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: fb8b8f5d-744d-4476-b6f6-918f6cd461ad kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: 'Created pod: model-chainer-raw-81bcf-f88d6597b-xj4kr' metadata: creationTimestamp: "2026-04-22T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: model-chainer-raw-81bcf-f88d6597b.18a8c6a68947e9e2 namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: a14dc0e2-82f0-48dd-ac17-9d685b267fa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 4c5ef70a-5a60-4981-9665-637687040ee9 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: Scaled up replica set model-chainer-raw-81bcf-f88d6597b from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:07:51Z" name: model-chainer-raw-81bcf.18a8c6a68864a2d6 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: c7250987-23e7-437c-b89c-36fa5bc6d330 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:07:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-81bcf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 6017d138-b45b-4980-bbd1-817cac876a33 kind: Event lastTimestamp: "2026-04-22T20:07:51Z" message: InferenceGraph [model-chainer-raw-81bcf] is Ready metadata: creationTimestamp: "2026-04-22T20:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:07:51Z" name: model-chainer-raw-81bcf.18a8c6a68bf8b0f7 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 9141ba01-51f9-4feb-8dee-fe10a0f1b995 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:09:21.821644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bb8cd58e1c namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 512fc718-cef8-4ae0-bbc0-7af681a44060 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-3e39c-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bb942b05f6 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 7936df1f-f322-4787-8cc9-01e95127f810 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:22Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:22Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:09:22Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bbc944b58c namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 50f2de49-2bd6-48f8-8289-302b8439bd21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e39c} kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c077214438a2f9bdb936ff2da330f1b072e635107d34dd2fdb9da07fac4424c5" already present on machine metadata: creationTimestamp: "2026-04-22T20:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:22Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bbca6f2009 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 59bc8477-419c-455d-8d8b-f533c70d3fcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e39c} kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:22Z" message: 'Created container: model-chainer-raw-hpa-3e39c' metadata: creationTimestamp: "2026-04-22T20:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:22Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bbcdd4143d namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: a0da0931-1197-47ac-8b04-275d32f9ff7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e39c} kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:22Z" message: Started container model-chainer-raw-hpa-3e39c metadata: creationTimestamp: "2026-04-22T20:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:22Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bbce41467f namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: c4af0b77-95f5-4f32-8dd0-4f75b2d25bf7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e39c} kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:31Z" message: Stopping container model-chainer-raw-hpa-3e39c metadata: creationTimestamp: "2026-04-22T20:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:31Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6bde42e575e namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 7e693ad4-dd32-470c-92ad-e3afe8da1913 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T20:09:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e39c} kind: Pod name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: c010b35c-b6e6-4b4e-ba52-1d9a65bc7391 kind: Event lastTimestamp: "2026-04-22T20:09:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T20:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:09:59Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4.18a8c6be74e25b41 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 96cb87a4-d3a4-4ae3-9eda-61ebe7ef177f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3e39c-565d6b7c6d namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 35d89159-2b64-4157-8070-bbf008b7631a kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: 'Created pod: model-chainer-raw-hpa-3e39c-565d6b7c6d-5xzs4' metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c-565d6b7c6d.18a8c6bb8c729630 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: c77178df-e0d4-4736-8d65-58fbcc8919fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 1764c139-c3c8-4889-8564-1dfe3d775514 kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: Scaled up replica set model-chainer-raw-hpa-3e39c-565d6b7c6d from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c.18a8c6bb8bc6c139 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 6fea57c1-6d7d-40bb-8a77-a3b6ab974586 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5abdd5e4-334a-404d-9e04-6e7aa61a1a6a kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: InferenceGraph [model-chainer-raw-hpa-3e39c] is Ready metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c.18a8c6bb8f5473c1 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 9f8fe31e-3b69-40d4-85f0-88393d3cb207 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 5abdd5e4-334a-404d-9e04-6e7aa61a1a6a kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-3e39c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-3e39c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c.18a8c6bb91465c09 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 2a8172bf-f990-4c94-9b0f-d5ec39353ec8 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:09:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3e39c namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 5abdd5e4-334a-404d-9e04-6e7aa61a1a6a kind: Event lastTimestamp: "2026-04-22T20:09:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-3e39c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:09:21Z" name: model-chainer-raw-hpa-3e39c.18a8c6bb9146a67d namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: d371e2a3-3316-4e2c-a06d-a0401cd048bc reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:14:48.970385Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18346" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-2bb0d-predictor-6755c856c-4flnf to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:14:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:14:48Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c707b86b3b2d namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: 34ea1b52-0d47-4108-a034-979be587d787 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:49Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:49Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:14:49Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c707d1fa05a7 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: c42ddd92-c6fd-4f1c-afc0-075b8e0f44fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:49Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c707d3312167 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: d3d8a86e-cbe5-493e-a01b-e6274c3e06e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:49Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c707d6511354 namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: 3a50a482-b9d1-48dd-9ff5-2e6437b14cad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:49Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c707d6bf24ec namespace: kserve-ci-e2e-test resourceVersion: "18374" uid: acd5df41-52fb-4113-97d9-52af285377db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:53Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:53Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c708b22ec8e9 namespace: kserve-ci-e2e-test resourceVersion: "18403" uid: be8a58a8-1455-4f2f-bcb8-1c2f3dd15893 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:53Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c708b59e8acf namespace: kserve-ci-e2e-test resourceVersion: "18407" uid: ca384baf-7b93-4399-833a-46c5d70212ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:14:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:14:53Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c708b63dc38a namespace: kserve-ci-e2e-test resourceVersion: "18409" uid: 84f42e99-7596-46ac-9d81-f1e0ccc32bba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T20:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:15:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:15:45Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c708ee528ab7 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 7251c691-8241-45eb-8a45-51a970d10a70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 33314084-ed1b-44e7-b322-fcc1910f2ac4 kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-2bb0d-predictor-6755c856c-4flnf.18a8c7180bad1ef2 namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: d3f1c78c-40d2-4098-b5c4-4d2cd55cabc2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-2bb0d-predictor-6755c856c namespace: kserve-ci-e2e-test resourceVersion: "18337" uid: b5c9ff1f-9eb6-49ba-bd50-5daf57c54367 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: 'Created pod: raw-sklearn-2bb0d-predictor-6755c856c-4flnf' metadata: creationTimestamp: "2026-04-22T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:48Z" name: raw-sklearn-2bb0d-predictor-6755c856c.18a8c707b7e8c1cb namespace: kserve-ci-e2e-test resourceVersion: "18347" uid: a28eaced-15cd-4bf9-8f8a-5268b0176066 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-2bb0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 18d5dd7e-1c68-426d-bf08-8a86e27eb4ae kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: Scaled up replica set raw-sklearn-2bb0d-predictor-6755c856c from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:14:48Z" name: raw-sklearn-2bb0d-predictor.18a8c707b56defe2 namespace: kserve-ci-e2e-test resourceVersion: "18338" uid: 5fb321d0-600f-4c4e-acbe-a0aa4ce73b10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:15:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2bb0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18336" uid: 48be90d8-6ea1-43d5-a69d-b99c74e457f8 kind: Event lastTimestamp: "2026-04-22T20:15:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:18Z" name: raw-sklearn-2bb0d-predictor.18a8c70b349978ef namespace: kserve-ci-e2e-test resourceVersion: "18597" uid: 687d6232-c5d5-4d13-b8ed-7315bc3a757d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:15:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2bb0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18336" uid: 48be90d8-6ea1-43d5-a69d-b99c74e457f8 kind: Event lastTimestamp: "2026-04-22T20:15:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:18Z" name: raw-sklearn-2bb0d-predictor.18a8c70b34f7022f namespace: kserve-ci-e2e-test resourceVersion: "18598" uid: d5cc3acc-544c-4e25-8bc2-1bac9704c283 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:15:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2bb0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18489" uid: 48be90d8-6ea1-43d5-a69d-b99c74e457f8 kind: Event lastTimestamp: "2026-04-22T20:15:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:15:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:48Z" name: raw-sklearn-2bb0d-predictor.18a8c7123301a025 namespace: kserve-ci-e2e-test resourceVersion: "18807" uid: 3956e8af-bce7-4147-9ecc-cde0ec098f78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:15:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2bb0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "18489" uid: 48be90d8-6ea1-43d5-a69d-b99c74e457f8 kind: Event lastTimestamp: "2026-04-22T20:15:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:15:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:48Z" name: raw-sklearn-2bb0d-predictor.18a8c7123360e266 namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: 82235305-ad37-4ccd-bcee-0c0141c3ad82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T20:14:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2bb0d namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 43a58c66-9ed9-4f65-9ba9-f19a37518545 kind: Event lastTimestamp: "2026-04-22T20:15:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:15:55Z" name: raw-sklearn-2bb0d.18a8c707b575ad5a namespace: kserve-ci-e2e-test resourceVersion: "18858" uid: ab2d5c90-cb39-48d2-88ae-33949a52e45f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:14:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2bb0d namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: 43a58c66-9ed9-4f65-9ba9-f19a37518545 kind: Event lastTimestamp: "2026-04-22T20:14:48Z" message: 'Failed to update status for InferenceService "raw-sklearn-2bb0d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-2bb0d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:14:48Z" name: raw-sklearn-2bb0d.18a8c707b60ba29e namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: 23a215ac-9354-4386-9171-ef4de6cc37fd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2bb0d namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 43a58c66-9ed9-4f65-9ba9-f19a37518545 kind: Event lastTimestamp: "2026-04-22T20:15:55Z" message: InferenceService [raw-sklearn-2bb0d] is Ready metadata: creationTimestamp: "2026-04-22T20:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:15:55Z" name: raw-sklearn-2bb0d.18a8c717262a66d2 namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: 7c4093d5-54a0-4b38-9ec5-835bec63afab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T20:15:59.131706Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh to ip-10-0-128-239.ec2.internal metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c7180e5bf714 namespace: kserve-ci-e2e-test resourceVersion: "18933" uid: b34cb9fa-83e6-440d-8ff9-4b69a1fd3ae0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-64998f957c-vr9sz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c71826f2e40b namespace: kserve-ci-e2e-test resourceVersion: "18945" uid: 32bad687-a67b-4dae-8a0b-1c98344e371b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a994fd91064f88b0a52b0b466ad083e0715e86891e056482a61038ab37cc2482" already present on machine metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c71828316faa namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: cc70ec2f-6e9e-404f-8eaf-7998e9309140 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c7182b43ca8f namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 312db6b7-761f-4c4d-9bba-010cf788ed48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c7182bb0ad5f namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: 6f0fb4fb-d81b-44dd-aaba-7814b4bc8e90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:16:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-22T20:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:16:04Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c71945a4efad namespace: kserve-ci-e2e-test resourceVersion: "18991" uid: cadecc73-7d32-41e5-9e29-58c4d72630d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:16:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T20:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:16:04Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c7194874b5ce namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: ade15f9d-5856-4a21-896c-14634460369f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:16:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T20:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:16:04Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c71948e08b27 namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 6ae919d5-6dba-4678-bf84-ae1fcc15017e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T20:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:16:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:16:56Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c71981e1239a namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 81d9045a-1ffa-4c96-854a-4212b3b93a4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:17:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 45a1b910-76ee-43e8-a2e9-4fe9f36611ec kind: Event lastTimestamp: "2026-04-22T20:17:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T20:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T20:17:09Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh.18a8c72861eb7f93 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: aa29832b-64f7-43e1-93ee-9cbba7d06717 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-239.ec2.internal source: component: kubelet host: ip-10-0-128-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9 namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: 89bd8c31-a62a-4c20-8a0a-8e9dd9bc09c8 kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: 'Created pod: raw-sklearn-runtime-e7bec-predictor-7ff77696d9-5jjbh' metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor-7ff77696d9.18a8c7180df4faf8 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 4fe81bfd-707c-472b-a409-e2172a0a14dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-e7bec-predictor namespace: kserve-ci-e2e-test resourceVersion: "18909" uid: e32c917e-a253-43aa-becf-e9f3cc727268 kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: Scaled up replica set raw-sklearn-runtime-e7bec-predictor-7ff77696d9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec-predictor.18a8c7180b13b0ff namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: d51fc513-d627-4482-ae07-08e38c4b2a6d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:16:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-e7bec-predictor namespace: kserve-ci-e2e-test resourceVersion: "18911" uid: 6fb854a7-a871-43c1-af93-52cc396c4acd kind: Event lastTimestamp: "2026-04-22T20:16:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:16:29Z" name: raw-sklearn-runtime-e7bec-predictor.18a8c71b89b52255 namespace: kserve-ci-e2e-test resourceVersion: "19173" uid: 70e557ae-1821-40eb-a927-efcd4747dd55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:16:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-e7bec-predictor namespace: kserve-ci-e2e-test resourceVersion: "18911" uid: 6fb854a7-a871-43c1-af93-52cc396c4acd kind: Event lastTimestamp: "2026-04-22T20:16:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T20:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:16:29Z" name: raw-sklearn-runtime-e7bec-predictor.18a8c71b8a149e83 namespace: kserve-ci-e2e-test resourceVersion: "19174" uid: 84ebdd2d-7da7-4753-becd-204a8df18124 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:16:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-e7bec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 6fb854a7-a871-43c1-af93-52cc396c4acd kind: Event lastTimestamp: "2026-04-22T20:16:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:16:59Z" name: raw-sklearn-runtime-e7bec-predictor.18a8c7228815adbb namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: 2be6bd2b-4804-42a3-938e-3912b4d25edd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T20:16:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-e7bec-predictor namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 6fb854a7-a871-43c1-af93-52cc396c4acd kind: Event lastTimestamp: "2026-04-22T20:16:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T20:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T20:16:59Z" name: raw-sklearn-runtime-e7bec-predictor.18a8c7228877e415 namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 77cb88f3-4d01-4bb0-8bd9-fa4e96188a7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-e7bec namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: d97f1b1e-d544-4fcd-b492-372d048cd6bc kind: Event lastTimestamp: "2026-04-22T20:17:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:17:06Z" name: raw-sklearn-runtime-e7bec.18a8c7180af15d03 namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: fb127714-046b-4083-bcc3-68d025e6d405 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:15:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-e7bec namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: d97f1b1e-d544-4fcd-b492-372d048cd6bc kind: Event lastTimestamp: "2026-04-22T20:15:59Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-e7bec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-e7bec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T20:15:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:15:59Z" name: raw-sklearn-runtime-e7bec.18a8c7180b8f6bc1 namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: 08ae5882-3ff4-4ced-83e4-5b73414a4cbb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T20:17:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-e7bec namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: d97f1b1e-d544-4fcd-b492-372d048cd6bc kind: Event lastTimestamp: "2026-04-22T20:17:06Z" message: InferenceService [raw-sklearn-runtime-e7bec] is Ready metadata: creationTimestamp: "2026-04-22T20:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T20:17:06Z" name: raw-sklearn-runtime-e7bec.18a8c727b9fdeccb namespace: kserve-ci-e2e-test resourceVersion: "19440" uid: 13b5a44c-eb67-4c95-81f7-f1c6b7c97d4c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "19669"