--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:54:11.226786Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18679" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3a34f54f79 namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: d0967879-9a1c-46c9-ba80-7410c45d18c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3a4f0f4d36 namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 18dd1904-d06d-4fb5-9be5-9b02d4e5de50 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: "2026-04-21T15:54:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:17Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3a50a4b4f6 namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 8dfbd792-139c-475e-a72e-21d10e7fee80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: "2026-04-21T15:54:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:17Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3a54812e57 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: 93ac60bf-3242-4c9b-98a5-3900ba452479 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: "2026-04-21T15:54:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:17Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3a54f99d0c namespace: kserve-ci-e2e-test resourceVersion: "18767" uid: a849b8e3-dfbb-46c9-845d-675d8b9f6a82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 48d460f5-eb59-4977-a544-3c3a8107b8d0 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-e57b3c-predictor-845664f98f-bsl29_kserve-ci-e2e-test(48d460f5-eb59-4977-a544-3c3a8107b8d0) metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:21Z" name: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29.18a86a3c848c1f2e namespace: kserve-ci-e2e-test resourceVersion: "18803" uid: 375eff5d-8d87-4dee-a206-e168feb66998 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-e57b3c-predictor-845664f98f namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: efe9c395-816c-47e5-9ec5-68a614b93029 kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: 'Created pod: isvc-init-fail-e57b3c-predictor-845664f98f-bsl29' metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c-predictor-845664f98f.18a86a3a3479ca79 namespace: kserve-ci-e2e-test resourceVersion: "18685" uid: fb624f4f-0b92-4677-8bfd-df7b1a9f47a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-e57b3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18662" uid: 48a617e2-182a-4b1f-bb7d-6f19d0392175 kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: Scaled up replica set isvc-init-fail-e57b3c-predictor-845664f98f from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c-predictor.18a86a3a30729510 namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: cb72b79d-3c05-43b5-98ea-136660ab7d2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e57b3c namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: fa6c6018-41c2-40c9-8fac-27d349aa085d kind: Event lastTimestamp: "2026-04-21T15:54:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:17Z" name: isvc-init-fail-e57b3c.18a86a3a306a0d37 namespace: kserve-ci-e2e-test resourceVersion: "18770" uid: f8dfddda-8b95-46cc-bc37-dae11832ebd8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e57b3c namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: fa6c6018-41c2-40c9-8fac-27d349aa085d kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: 'Failed to update status for InferenceService "isvc-init-fail-e57b3c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e57b3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c.18a86a3a31307f58 namespace: kserve-ci-e2e-test resourceVersion: "18692" uid: 81a299ae-959f-4143-b7cf-c69e70827583 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e57b3c namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: fa6c6018-41c2-40c9-8fac-27d349aa085d kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e57b3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:11Z" name: isvc-init-fail-e57b3c.18a86a3a3130b697 namespace: kserve-ci-e2e-test resourceVersion: "18693" uid: 838b3dd5-5a2e-4609-b1b4-ece2b19c77c0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:48:27.197621Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869ea1b38d14c namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: c1067142-8ae4-4134-8267-39e575e32657 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869ea34d02fcb namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: a32b5252-14b1-48d4-9f6b-dc4300d437d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869ea369a62da namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 5b0eef8f-5b11-4e59-8343-58aaed340b1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869ea39e7f5be namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: ac6329ad-d063-42fd-9768-7c00cb082505 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869ea3a606318 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: a3a3ddc6-5aaf-4e10-9bde-ddc3f083d77c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:31Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb31e19ccd namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: dbfdbead-527f-452c-802e-c28353227271 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:31Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb3546d844 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 4b795525-3aac-4c2a-bb81-5e53cf74108c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:31Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb35cbb4f8 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 827472fc-a49b-4c51-8fd4-14ddc500e4cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:31Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:56140db5c247b351ac2dad1a46e24a34c7c91836ee8418891bd73409e3a23542" already present on machine metadata: creationTimestamp: "2026-04-21T15:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:31Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb35d9a6a2 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 4a8374e2-6875-43f3-8b43-f7b271ddc007 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:31Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:31Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb39527012 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: c32fa9c1-4107-460c-ad1e-6bb2eb157e26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:48:32Z" message: Started container agent metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:32Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb39c463de namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 7ba8e24a-7c5d-42f5-90bd-b9a5fc797ead reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-21T15:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:50:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:50:19Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb6ddce741 namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: b139e77c-9598-4f5d-9eb8-5ac70dedc61c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-21T15:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:50:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:50:19Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869eb6de706ea namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 929a82c9-f356-413f-ac4f-f21e4939c177 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:52Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869fdf98010a5 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: a28bb400-ae03-4f95-9943-40d5c230876d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d0b5fd60-7400-4218-9e2c-f8f3ea2ef9c9 kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:52Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm.18a869fdf9818c36 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 7a2c5a74-0511-45a4-aa0c-3eb0eb345049 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-61a06-predictor-58dddf4bf8 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: ad9aeb93-e6b3-490a-9bbc-31c4f2af1694 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: 'Created pod: isvc-logger-raw-61a06-predictor-58dddf4bf8-8wlzm' metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor-58dddf4bf8.18a869ea1abd3a33 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: c0f940a9-69a6-45b7-8fe8-b56c662f285b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 5c241089-b163-47f8-b3c3-6fc768c2e037 kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: Scaled up replica set isvc-logger-raw-61a06-predictor-58dddf4bf8 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06-predictor.18a869ea14e98ea4 namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 969f273a-4dbd-4db3-a6d7-81250befa2ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 194d83cb-caa1-4bc9-ba27-9a84b4d9d42e kind: Event lastTimestamp: "2026-04-21T15:48:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:42Z" name: isvc-logger-raw-61a06-predictor.18a869ed93995536 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: f8c04921-08d0-4d25-91c5-04da4a8448ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 194d83cb-caa1-4bc9-ba27-9a84b4d9d42e kind: Event lastTimestamp: "2026-04-21T15:48:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:42Z" name: isvc-logger-raw-61a06-predictor.18a869ed9412e26a namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 833d50ae-30d0-4b17-9adf-a049ddad1ba5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:48:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 194d83cb-caa1-4bc9-ba27-9a84b4d9d42e kind: Event lastTimestamp: "2026-04-21T15:49:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:49:42Z" name: isvc-logger-raw-61a06-predictor.18a869f11370c4b8 namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: db761f61-0d18-4575-b767-cb2c6c9e1990 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:48:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 194d83cb-caa1-4bc9-ba27-9a84b4d9d42e kind: Event lastTimestamp: "2026-04-21T15:49:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:49:42Z" name: isvc-logger-raw-61a06-predictor.18a869f113e9ae66 namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 3633bb68-87bb-4fef-b234-a1d74d860562 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2bd183a7-c183-4299-aab4-4c0f6ee3e42f kind: Event lastTimestamp: "2026-04-21T15:49:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:49:39Z" name: isvc-logger-raw-61a06.18a869ea14cbcd8f namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 37882dd5-37ad-400b-933d-ff35c6181d99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2bd183a7-c183-4299-aab4-4c0f6ee3e42f kind: Event lastTimestamp: "2026-04-21T15:48:27Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-61a06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-61a06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:27Z" name: isvc-logger-raw-61a06.18a869ea15779303 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 75799d7b-2974-451f-8813-928101be3552 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 2bd183a7-c183-4299-aab4-4c0f6ee3e42f kind: Event lastTimestamp: "2026-04-21T15:49:39Z" message: InferenceService [isvc-logger-raw-61a06] is Ready metadata: creationTimestamp: "2026-04-21T15:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:49:39Z" name: isvc-logger-raw-61a06.18a869fb070d286a namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: ba50787d-71ae-4b57-86b7-ecd0c4eb1001 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:52:32.783270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17823" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-daad18-predictor-7d747cb478-h4pvv to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:52:32Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a2349446098 namespace: kserve-ci-e2e-test resourceVersion: "17829" uid: cea4266f-9dd7-4195-bba4-177a43a66b90 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17834" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:33Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:52:33Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a2362b5d88f namespace: kserve-ci-e2e-test resourceVersion: "17839" uid: 9b3a2df1-87b8-4982-b373-f1ca7c42cb61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:33Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a23641bd42e namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: 50e62a65-755a-4b09-b0ba-456c8f3da1c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:33Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a2367062273 namespace: kserve-ci-e2e-test resourceVersion: "17843" uid: 0ddf3bea-9998-4b65-bf97-da928623b5fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:33Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a236779b8c9 namespace: kserve-ci-e2e-test resourceVersion: "17844" uid: 094ec750-f575-45b6-bc25-b93d948b5520 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:37Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a2471be7dd3 namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 18e80847-4eb8-42a5-948f-74960071c21f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:37Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a247515c938 namespace: kserve-ci-e2e-test resourceVersion: "17891" uid: 2855da26-b8b0-4567-ba00-4dbaa96ff780 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:52:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:37Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a247592a9e3 namespace: kserve-ci-e2e-test resourceVersion: "17892" uid: c936f56a-9ad6-4aca-a969-32dffbbcb820 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:52:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:53:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:53:39Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a24adb0ad5b namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: 5cf31bf8-e42e-4b88-8e50-572cd7066e22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-daad18-predictor-7d747cb478-h4pvv namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 34654724-da9c-44af-a61c-ed4883f3999d kind: Event lastTimestamp: "2026-04-21T15:54:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:11Z" name: isvc-primary-daad18-predictor-7d747cb478-h4pvv.18a86a3a2bc71167 namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: 430cdd21-bd8d-4555-9969-23b5268ba4a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-daad18-predictor-7d747cb478 namespace: kserve-ci-e2e-test resourceVersion: "17813" uid: d37e9bc3-bd40-40ff-8889-01e5d3febe4a kind: Event lastTimestamp: "2026-04-21T15:52:32Z" message: 'Created pod: isvc-primary-daad18-predictor-7d747cb478-h4pvv' metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:32Z" name: isvc-primary-daad18-predictor-7d747cb478.18a86a2348a45b7b namespace: kserve-ci-e2e-test resourceVersion: "17824" uid: 26f5dbb4-bed3-4174-b5b3-759fd7ad900d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "17809" uid: ad75bbff-ee86-41f4-82d9-51ee101877a2 kind: Event lastTimestamp: "2026-04-21T15:52:32Z" message: Scaled up replica set isvc-primary-daad18-predictor-7d747cb478 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:32Z" name: isvc-primary-daad18-predictor.18a86a234414b102 namespace: kserve-ci-e2e-test resourceVersion: "17814" uid: 85155c63-a950-4442-b48b-8f36cf08d203 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: defaf977-f7ef-4029-8bbd-0d9ebed9777c kind: Event lastTimestamp: "2026-04-21T15:52:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:52:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:47Z" name: isvc-primary-daad18-predictor.18a86a26c2dec788 namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: c3ffeeca-0d94-4ac7-bfbc-622a807f462c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: defaf977-f7ef-4029-8bbd-0d9ebed9777c kind: Event lastTimestamp: "2026-04-21T15:52:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:52:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:47Z" name: isvc-primary-daad18-predictor.18a86a26c365bab7 namespace: kserve-ci-e2e-test resourceVersion: "17965" uid: a85f3cbf-7fb6-4b1a-b361-1893af79e9ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-21T15:53:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: defaf977-f7ef-4029-8bbd-0d9ebed9777c kind: Event lastTimestamp: "2026-04-21T15:54:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:53:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:02Z" name: isvc-primary-daad18-predictor.18a86a2a4352dbe6 namespace: kserve-ci-e2e-test resourceVersion: "18563" uid: bfa78f6c-25f5-440d-8e1c-1ebc42a59246 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-21T15:53:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: defaf977-f7ef-4029-8bbd-0d9ebed9777c kind: Event lastTimestamp: "2026-04-21T15:54:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:53:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:02Z" name: isvc-primary-daad18-predictor.18a86a2a43dde5c7 namespace: kserve-ci-e2e-test resourceVersion: "18564" uid: 1d98df20-d9e6-4f20-87b1-50e78361f302 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: b942fd6a-bf8f-4f7b-8bc3-827b918d4a8e kind: Event lastTimestamp: "2026-04-21T15:53:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:53:49Z" name: isvc-primary-daad18.18a86a23441b3378 namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: a657f1bb-1068-422d-83fe-3c3fa0ea002d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: b942fd6a-bf8f-4f7b-8bc3-827b918d4a8e kind: Event lastTimestamp: "2026-04-21T15:52:32Z" message: 'Failed to update status for InferenceService "isvc-primary-daad18": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-daad18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:52:32Z" name: isvc-primary-daad18.18a86a2344b9b117 namespace: kserve-ci-e2e-test resourceVersion: "17818" uid: b0448c91-3586-4fb4-8d15-7f7261367af2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: b942fd6a-bf8f-4f7b-8bc3-827b918d4a8e kind: Event lastTimestamp: "2026-04-21T15:52:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-daad18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:52:32Z" name: isvc-primary-daad18.18a86a2344b9ffbd namespace: kserve-ci-e2e-test resourceVersion: "17819" uid: 802898b9-e067-4719-a2c4-2b601827dbbe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "18403" uid: b942fd6a-bf8f-4f7b-8bc3-827b918d4a8e kind: Event lastTimestamp: "2026-04-21T15:53:49Z" message: InferenceService [isvc-primary-daad18] is Ready metadata: creationTimestamp: "2026-04-21T15:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:53:49Z" name: isvc-primary-daad18.18a86a353a4cb8e1 namespace: kserve-ci-e2e-test resourceVersion: "18404" uid: 9b3fb488-9322-4c97-ab8e-b90d7a738984 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:43:01.024704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12920" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:43:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699e29cd1906 namespace: kserve-ci-e2e-test resourceVersion: "12924" uid: 321f9b36-80a5-496b-814b-f206b9faf26a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12927" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:01Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699e4361e455 namespace: kserve-ci-e2e-test resourceVersion: "12940" uid: be1e1e98-6407-4f27-a3b0-02f4dc78a569 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" metadata: creationTimestamp: "2026-04-21T15:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699e44f101d8 namespace: kserve-ci-e2e-test resourceVersion: "12942" uid: e959507b-9730-4bf4-a31b-ddf5ea90cc49 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:05Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" in 4.144s (4.144s including waiting). Image size: 301280147 bytes.' metadata: creationTimestamp: "2026-04-21T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:05Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699f3bf7478e namespace: kserve-ci-e2e-test resourceVersion: "12976" uid: 9a46990a-c39c-445a-9e9a-c3f11e912088 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:05Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699f3fc199eb namespace: kserve-ci-e2e-test resourceVersion: "12977" uid: 79342717-d57b-4e99-97a1-fcc48e5cbf22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:05Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a8699f4029f70f namespace: kserve-ci-e2e-test resourceVersion: "12978" uid: 085af530-6098-404c-aa7f-c9eba5e02f5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" metadata: creationTimestamp: "2026-04-21T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:09Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a02c0b85f8 namespace: kserve-ci-e2e-test resourceVersion: "13012" uid: d5adffbc-5161-479f-8370-60af40acc4be reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" in 13.409s (13.409s including waiting). Image size: 1560596387 bytes.' metadata: creationTimestamp: "2026-04-21T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:23Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a34b4caf2d namespace: kserve-ci-e2e-test resourceVersion: "13107" uid: 7e882bbc-644a-4297-b251-83596cb66c82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:23Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a34ea11728 namespace: kserve-ci-e2e-test resourceVersion: "13108" uid: d62f71b4-d119-4d64-991d-111804846437 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:23Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a34f11c5b5 namespace: kserve-ci-e2e-test resourceVersion: "13109" uid: 02e40a5a-be6c-4983-9894-c19c3a903ba6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:23Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:56140db5c247b351ac2dad1a46e24a34c7c91836ee8418891bd73409e3a23542" metadata: creationTimestamp: "2026-04-21T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:23Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a34f1e7982 namespace: kserve-ci-e2e-test resourceVersion: "13110" uid: 9c267395-9086-4b3a-beb2-6f033d0207f1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:56140db5c247b351ac2dad1a46e24a34c7c91836ee8418891bd73409e3a23542" in 2.15s (2.15s including waiting). Image size: 237897782 bytes.' metadata: creationTimestamp: "2026-04-21T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:25Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a3cf4d4f2b namespace: kserve-ci-e2e-test resourceVersion: "13127" uid: e5ff1f99-7f4d-4d64-8335-b7aeec869cc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:25Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-21T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:25Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a3d26ab748 namespace: kserve-ci-e2e-test resourceVersion: "13128" uid: 20d75a1c-0cae-4b8d-8e71-7ff0229902f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:43:25Z" message: Started container agent metadata: creationTimestamp: "2026-04-21T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:43:25Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a3d2de3aec namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: 88e55f29-ea44-4394-89a3-332415c50faf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-21T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:45:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:10Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a3ea0a513a namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 5d3ae37f-1967-4a55-8918-e9c872053596 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-21T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:45:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-21T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:10Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869a425d8e48f namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 0104b260-b1ce-40c4-a114-2d6fbb5c6413 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:46Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869b6ad3fc4d2 namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: a8aa3e38-b271-47dc-97a1-8e88e477f81f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 88819024-ef88-4177-9ec6-3d7b2fe18065 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:46Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj.18a869b6ad402656 namespace: kserve-ci-e2e-test resourceVersion: "13749" uid: d36a9c78-d6b9-472d-932c-50f5a4f6d15d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: ea58f160-235f-486c-a42a-ce0b24c336a9 kind: Event lastTimestamp: "2026-04-21T15:43:01Z" message: 'Created pod: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c-67tvj' metadata: creationTimestamp: "2026-04-21T15:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c.18a8699e2930bc76 namespace: kserve-ci-e2e-test resourceVersion: "12921" uid: 067ed0b9-3648-4e35-9623-80c334440977 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:43:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-03d5d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12904" uid: 74acecd1-2ade-478a-95e0-82f0fc8c89fc kind: Event lastTimestamp: "2026-04-21T15:43:00Z" message: Scaled up replica set isvc-raw-sklearn-batcher-03d5d-predictor-6c5d487c5c from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:43:00Z" name: isvc-raw-sklearn-batcher-03d5d-predictor.18a8699e1f93ac66 namespace: kserve-ci-e2e-test resourceVersion: "12906" uid: dd0a1f28-4f22-4532-a7ea-30774482a3bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03d5d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12913" uid: 6c91f700-93c4-46f9-8c24-7f8f378ae50d kind: Event lastTimestamp: "2026-04-21T15:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor.18a869a1a58fcbdf namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: 351b78e1-1ce7-4b1f-816d-8965e953ea02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03d5d-predictor namespace: kserve-ci-e2e-test resourceVersion: "12913" uid: 6c91f700-93c4-46f9-8c24-7f8f378ae50d kind: Event lastTimestamp: "2026-04-21T15:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:01Z" name: isvc-raw-sklearn-batcher-03d5d-predictor.18a869a1a60a4556 namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: fd8ad3c6-d983-4d89-9ecb-5b744f6eb2cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03d5d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13058" uid: 6c91f700-93c4-46f9-8c24-7f8f378ae50d kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-raw-sklearn-batcher-03d5d-predictor.18a869afa39ecb9e namespace: kserve-ci-e2e-test resourceVersion: "13730" uid: 02e8706f-49cf-44ef-aafe-ec3c05634f10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03d5d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13058" uid: 6c91f700-93c4-46f9-8c24-7f8f378ae50d kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-raw-sklearn-batcher-03d5d-predictor.18a869afa425682f namespace: kserve-ci-e2e-test resourceVersion: "13735" uid: bd74f596-fa7f-4976-8e47-758d7362442f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-21T15:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03d5d namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 9f8068d7-e593-4ae4-8f4a-3d13d7197d72 kind: Event lastTimestamp: "2026-04-21T15:44:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:44:31Z" name: isvc-raw-sklearn-batcher-03d5d.18a8699e25aa4b8e namespace: kserve-ci-e2e-test resourceVersion: "13612" uid: dcd034f8-aad5-4d5b-819d-81d23318dc65 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03d5d namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 9f8068d7-e593-4ae4-8f4a-3d13d7197d72 kind: Event lastTimestamp: "2026-04-21T15:43:01Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-03d5d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-03d5d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d.18a8699e26164b8d namespace: kserve-ci-e2e-test resourceVersion: "12930" uid: 9d85f37a-e798-4f74-9b0e-4583f7123fbf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03d5d namespace: kserve-ci-e2e-test resourceVersion: "12893" uid: 9f8068d7-e593-4ae4-8f4a-3d13d7197d72 kind: Event lastTimestamp: "2026-04-21T15:43:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-03d5d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:43:01Z" name: isvc-raw-sklearn-batcher-03d5d.18a8699e2616918b namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 7cba9431-4c4d-4c93-b184-3fd161c043d9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03d5d namespace: kserve-ci-e2e-test resourceVersion: "13610" uid: 9f8068d7-e593-4ae4-8f4a-3d13d7197d72 kind: Event lastTimestamp: "2026-04-21T15:44:31Z" message: InferenceService [isvc-raw-sklearn-batcher-03d5d] is Ready metadata: creationTimestamp: "2026-04-21T15:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:44:31Z" name: isvc-raw-sklearn-batcher-03d5d.18a869b31e711531 namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: 063fd868-c276-4a4e-98ae-93b17ee6560c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:53:52.979129Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-daad18-predictor-58999cdf79-vzkg4 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:53:52Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a35f5501752 namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: d81c2c9f-6971-442f-8def-e01aaacff143 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: "2026-04-21T15:53:53Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:53:53Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a360fafeb36 namespace: kserve-ci-e2e-test resourceVersion: "18475" uid: 59d50a05-3d33-4191-9829-55edafa59a72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: "2026-04-21T15:53:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:53:59Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a3611bb5c4d namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: a02c4809-4f30-4746-87f0-a947654b7bf2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: "2026-04-21T15:53:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:53:59Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a361600deed namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: f840913f-e240-4df3-9bd4-4658f3594cc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: "2026-04-21T15:53:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:53:59Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a361675c435 namespace: kserve-ci-e2e-test resourceVersion: "18530" uid: 73df19ad-0315-4475-ac82-021a4d0fd595 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4 namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: f334e01d-09cc-425c-8ef3-0d3cbff4f175 kind: Event lastTimestamp: "2026-04-21T15:54:02Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-daad18-predictor-58999cdf79-vzkg4_kserve-ci-e2e-test(f334e01d-09cc-425c-8ef3-0d3cbff4f175) metadata: creationTimestamp: "2026-04-21T15:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:02Z" name: isvc-secondary-daad18-predictor-58999cdf79-vzkg4.18a86a38132694d8 namespace: kserve-ci-e2e-test resourceVersion: "18555" uid: 10bceb42-36e2-464d-92c0-6d4256341f3d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-daad18-predictor-58999cdf79 namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: c700d74b-4dcd-4ae1-9c5d-0c0f8ad9b29d kind: Event lastTimestamp: "2026-04-21T15:53:52Z" message: 'Created pod: isvc-secondary-daad18-predictor-58999cdf79-vzkg4' metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:53:52Z" name: isvc-secondary-daad18-predictor-58999cdf79.18a86a35f4ab2764 namespace: kserve-ci-e2e-test resourceVersion: "18459" uid: 2dc02a56-7e72-432b-9f2b-25a06b692f81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: 624eecfe-c0fe-4071-a6df-29f5a6ab045d kind: Event lastTimestamp: "2026-04-21T15:53:52Z" message: Scaled up replica set isvc-secondary-daad18-predictor-58999cdf79 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:53:52Z" name: isvc-secondary-daad18-predictor.18a86a35f08212fc namespace: kserve-ci-e2e-test resourceVersion: "18451" uid: 6b0f1482-ea7b-4559-b677-a8ced6ecfc53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 3146610b-ca9e-4d5c-8f37-d57b24ff44db kind: Event lastTimestamp: "2026-04-21T15:54:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:07Z" name: isvc-secondary-daad18-predictor.18a86a396fc0a96d namespace: kserve-ci-e2e-test resourceVersion: "18601" uid: 9c97e1f3-b76f-4c33-97e5-7d6fc038ba92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-daad18-predictor namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 3146610b-ca9e-4d5c-8f37-d57b24ff44db kind: Event lastTimestamp: "2026-04-21T15:54:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:07Z" name: isvc-secondary-daad18-predictor.18a86a3970389449 namespace: kserve-ci-e2e-test resourceVersion: "18602" uid: 4c84e470-c419-4884-9ed3-39194e5c20b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-21T15:53:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: ce672721-cc85-4479-a0cf-0db9dc4d3afd kind: Event lastTimestamp: "2026-04-21T15:54:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:02Z" name: isvc-secondary-daad18.18a86a35f04a234b namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: ee346782-c10a-4427-831f-ddf25cde9b33 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: ce672721-cc85-4479-a0cf-0db9dc4d3afd kind: Event lastTimestamp: "2026-04-21T15:53:52Z" message: 'Failed to update status for InferenceService "isvc-secondary-daad18": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-daad18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:53:52Z" name: isvc-secondary-daad18.18a86a35f0bcd874 namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: ebc72071-a789-40e2-a93b-54d0963a9229 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:53:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-daad18 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: ce672721-cc85-4479-a0cf-0db9dc4d3afd kind: Event lastTimestamp: "2026-04-21T15:53:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-daad18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:53:52Z" name: isvc-secondary-daad18.18a86a35f0bd1dd1 namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: b65fac25-82cb-43af-ab2e-5e48818d99b4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:44:46.650828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:44:46Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b6c19bbb6a namespace: kserve-ci-e2e-test resourceVersion: "13775" uid: a77d6c57-82e0-4450-a1c3-47aa4097e4df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:44:47Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b6db3b4372 namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: 766cc91b-cad2-4c63-ad46-91f096509db1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b6de50b521 namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: c8f19932-a2bd-481f-8f60-90c0fa146b6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b6e2095fb0 namespace: kserve-ci-e2e-test resourceVersion: "13820" uid: 0aec82ea-ffda-460e-86e1-52d2b3734601 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b6e28141e7 namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: e9c0d9df-099f-49bb-a8b3-3f454ad9230b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:52Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b8032c251d namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: d27f466d-bf10-4030-a744-bc8efb37527d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:52Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b806393798 namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: ca47b948-d1e4-464d-be9c-9a80d6559694 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:44:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:52Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b806b264d4 namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: b8c0a856-fb6c-4e2f-93a0-04a20720922f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:45:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:54Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869b83f4e3ce6 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 0765027c-130b-412d-963b-62c380eb12a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb namespace: kserve-ci-e2e-test resourceVersion: "13768" uid: 1e99a0f4-3071-46ed-a767-89c498271f7e kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:26Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb.18a869ce0d9893ff namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: e0bc2ced-fdb0-42ac-b33a-23217ce31408 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67 namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: b3e25dc3-e027-4b10-88a1-d731ff6c51b0 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'Created pod: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67-q49pb' metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67.18a869b6c04feb18 namespace: kserve-ci-e2e-test resourceVersion: "13771" uid: 3089019c-14c5-40c2-a808-b9d1cb2b1602 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: df559096-f0cf-47d1-8a15-1b824b85ddd8 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: Scaled up replica set isvc-sklearn-graph-raw-4bd9d-predictor-85f57b9f67 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-sklearn-graph-raw-4bd9d-predictor.18a869b6b09f091f namespace: kserve-ci-e2e-test resourceVersion: "13757" uid: c81fc5c7-a712-460e-88f2-ac8a92a8c6a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: 7b647092-0ee7-47cc-b000-3f9476cb6621 kind: Event lastTimestamp: "2026-04-21T15:45:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:45:31Z" name: isvc-sklearn-graph-raw-4bd9d-predictor.18a869ba3230fd76 namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: f9064f59-a753-49cc-ab8c-6c91986acdbc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: 7b647092-0ee7-47cc-b000-3f9476cb6621 kind: Event lastTimestamp: "2026-04-21T15:45:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:45:31Z" name: isvc-sklearn-graph-raw-4bd9d-predictor.18a869ba32fcbbe2 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 0f861203-203e-4fa2-a3c9-0af99a600f21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13956" uid: 7b647092-0ee7-47cc-b000-3f9476cb6621 kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: isvc-sklearn-graph-raw-4bd9d-predictor.18a869c4b19b6338 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 7122a8ff-9987-4b73-9a91-9118accc5bcf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13956" uid: 7b647092-0ee7-47cc-b000-3f9476cb6621 kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: isvc-sklearn-graph-raw-4bd9d-predictor.18a869c4b2278e05 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: b8c7b75c-2ae7-4334-93e1-b39226ebb046 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "13729" uid: fd985f2d-6c44-45c6-8205-ee7ad50caa13 kind: Event lastTimestamp: "2026-04-21T15:46:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:46:04Z" name: isvc-sklearn-graph-raw-4bd9d.18a869b6b09e8848 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: a066e138-fbd4-4414-bd2f-7d820168536a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "13729" uid: fd985f2d-6c44-45c6-8205-ee7ad50caa13 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-4bd9d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-4bd9d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-sklearn-graph-raw-4bd9d.18a869b6b1e1e242 namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: f3927163-790f-4f9d-8111-95e4f92fd264 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: fd985f2d-6c44-45c6-8205-ee7ad50caa13 kind: Event lastTimestamp: "2026-04-21T15:46:04Z" message: InferenceService [isvc-sklearn-graph-raw-4bd9d] is Ready metadata: creationTimestamp: "2026-04-21T15:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:46:04Z" name: isvc-sklearn-graph-raw-4bd9d.18a869c8ccb66ac6 namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: c643cc47-c10b-42cf-aa0c-cc0d67491d6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:46:26.758144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:46:26Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869ce1078327c namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: de1b9571-b5ba-4ebf-b652-d5a234d71223 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:46:27Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869ce2af6abb1 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: d7d83338-5544-4ab9-8d58-7c6219bdf4f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869ce2c889600 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 93556325-af0c-4deb-ae89-ad1aa16088c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869ce30a8a7a5 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 9ac066da-54d4-48f3-8c5c-7135476793db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869ce312b840f namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 7e8c3bdd-b14f-4b41-9f8a-3896d7523a38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869cf26a8ea53 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: c51b21ad-99e0-432f-9888-dab10e4ace6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869cf2a3d05fd namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: bd58001f-89e2-436a-8a87-ce6fb43ba514 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869cf2ad0bc80 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: da3ecdaa-2678-4929-8127-a0d3ffb30d0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-21T15:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:47:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:47:36Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869cf62cf08b2 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 6486318a-09ff-45e6-82b1-ee0bfcd75be6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bf28ca68-7e3c-46d1-8877-a58305cadd3b kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:17Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4.18a869e7bee8823a namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: fdf4b82b-6b2a-468c-bfc1-dc569a55ed43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 3c0e1a37-4e2a-498d-ba44-fe494b280924 kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f-xkfv4' metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:26Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f.18a869ce1007b27a namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: fda6ae1a-e189-4d7e-831e-08f2ac7c7db4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 0a8c346a-db6f-42d4-ad66-451be47c6c07 kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3991c-predictor-65d954f97f from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:26Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor.18a869ce0bd92cec namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 6c2093bd-0ff7-441c-bfd4-8dc876e263ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 0cd8c003-5f82-4cb7-99df-00a2ea01bfad kind: Event lastTimestamp: "2026-04-21T15:47:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:47:11Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor.18a869d18a738ddf namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: e89d06bf-727e-42c3-8c7d-514749a32ec9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 0cd8c003-5f82-4cb7-99df-00a2ea01bfad kind: Event lastTimestamp: "2026-04-21T15:47:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:47:11Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor.18a869d18aeaf67e namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 19bb5d3c-5203-4603-aef6-081e52cbdeb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:47:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 0cd8c003-5f82-4cb7-99df-00a2ea01bfad kind: Event lastTimestamp: "2026-04-21T15:48:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:11Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor.18a869dc08b9c5b3 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 38056c3b-72e7-4993-b601-055eeba9a59f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:47:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 0cd8c003-5f82-4cb7-99df-00a2ea01bfad kind: Event lastTimestamp: "2026-04-21T15:48:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:11Z" name: isvc-sklearn-graph-raw-hpa-3991c-predictor.18a869dc092da6b7 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 2cbabaf9-9650-46fb-9a62-fc71f0c875d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 659bad85-3175-4e3b-9f2c-fa18746d725a kind: Event lastTimestamp: "2026-04-21T15:47:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:47:46Z" name: isvc-sklearn-graph-raw-hpa-3991c.18a869ce0bbaf6a3 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 0cd3e0cf-0f65-4325-a2bb-390d5add5cc5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 659bad85-3175-4e3b-9f2c-fa18746d725a kind: Event lastTimestamp: "2026-04-21T15:47:46Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3991c] is Ready metadata: creationTimestamp: "2026-04-21T15:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:47:46Z" name: isvc-sklearn-graph-raw-hpa-3991c.18a869e0b85e0a72 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: bd4a45e4-2f1b-4d2b-9158-97d6de7f4b90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:49:52.595208Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:49:52Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869fdfd508a3f namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: bacf87ab-631b-485c-a3b8-bd3d24ab6e19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:53Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:49:53Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869fe18036dfc namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: cc029792-2ae4-4ed8-9026-71fade36b236 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:53Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869fe19ae0e2e namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: fc70c5fc-44ab-40fa-9d72-f104bb1b7b1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:53Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869fe1c8ee6ac namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: f95dd989-b6ad-4792-9018-aa9489ef7c53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:53Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869fe1d06a006 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 98bf01a3-0c7b-40ac-a7ca-a03c5ec9f9ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:57Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869ff0ea0d6f1 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 7647ff35-c94d-40fa-ab74-1f51ba82ce67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:57Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869ff1232da7b namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: c558257f-c300-4632-9330-c234cf71b2cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:49:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:57Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869ff12aebb54 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: a06107df-bdf7-4ab5-8ffb-a00064c2a138 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-21T15:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:52:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:35Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a869ff4a8e3e1c namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 76884959-6d2f-490d-97cf-6562247ac20c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: bbf23fac-035d-482a-bbbd-0e1761f28b94 kind: Event lastTimestamp: "2026-04-21T15:52:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:52:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:52:32Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8.18a86a234410581f namespace: kserve-ci-e2e-test resourceVersion: "17815" uid: 406f087e-9b13-4820-9656-ea9cf3481d63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 97cb2b7b-a509-4e15-9eae-788968dc8554 kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: 'Created pod: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f-d24z8' metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:49:52Z" name: isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f.18a869fdfcb7e7b6 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 86d0411d-c770-4c9a-badc-258744cb6f30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-1f208-predictor namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: ad7b88fa-ca9f-437f-a310-b6ffd3fbf392 kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: Scaled up replica set isvc-sklearn-scale-raw-1f208-predictor-55ff5c688f from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:49:52Z" name: isvc-sklearn-scale-raw-1f208-predictor.18a869fdf50630f1 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: f510deac-b355-4e53-b700-fa01463557f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1f208-predictor namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 2558671f-a4b5-45b4-a684-e4cc09f146e8 kind: Event lastTimestamp: "2026-04-21T15:50:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:50:22Z" name: isvc-sklearn-scale-raw-1f208-predictor.18a86a0173ba2955 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: 6e3b0dbd-b179-4927-9b90-d7d6b196386c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1f208-predictor namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 2558671f-a4b5-45b4-a684-e4cc09f146e8 kind: Event lastTimestamp: "2026-04-21T15:50:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:50:22Z" name: isvc-sklearn-scale-raw-1f208-predictor.18a86a01742d186c namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 8f82fb3e-e003-4a38-9c19-a8781c765fcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:50:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1f208-predictor namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 2558671f-a4b5-45b4-a684-e4cc09f146e8 kind: Event lastTimestamp: "2026-04-21T15:52:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:22Z" name: isvc-sklearn-scale-raw-1f208-predictor.18a86a08725eb4e4 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: 61bcf479-50f8-448e-83a1-e874dbd79166 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:50:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1f208-predictor namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 2558671f-a4b5-45b4-a684-e4cc09f146e8 kind: Event lastTimestamp: "2026-04-21T15:52:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:52:22Z" name: isvc-sklearn-scale-raw-1f208-predictor.18a86a0872dbdd6c namespace: kserve-ci-e2e-test resourceVersion: "17708" uid: 511abec7-f2b6-4396-befc-854fdbd82f8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1f208 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 4331471a-7732-4ea2-a16a-76b97727638c kind: Event lastTimestamp: "2026-04-21T15:52:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:52:25Z" name: isvc-sklearn-scale-raw-1f208.18a869fdf4de0e10 namespace: kserve-ci-e2e-test resourceVersion: "17738" uid: 999ac722-e39d-4166-bbdb-ed8de1d2ec95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1f208 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 4331471a-7732-4ea2-a16a-76b97727638c kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-1f208": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-1f208": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:49:52Z" name: isvc-sklearn-scale-raw-1f208.18a869fdfb0020e0 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 1667f317-6193-4876-941e-30b9e19ca8bb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:52:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1f208 namespace: kserve-ci-e2e-test resourceVersion: "17736" uid: 4331471a-7732-4ea2-a16a-76b97727638c kind: Event lastTimestamp: "2026-04-21T15:52:25Z" message: InferenceService [isvc-sklearn-scale-raw-1f208] is Ready metadata: creationTimestamp: "2026-04-21T15:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:52:25Z" name: isvc-sklearn-scale-raw-1f208.18a86a21aeaa13aa namespace: kserve-ci-e2e-test resourceVersion: "17737" uid: 06ea8146-2557-481d-897c-3e7441f58992 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:44:46.944685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13802" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:44:46Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b6d31fb684 namespace: kserve-ci-e2e-test resourceVersion: "13810" uid: 86aaceee-2960-4016-8585-78540b18e778 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13813" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:44:47Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b6ec919ca1 namespace: kserve-ci-e2e-test resourceVersion: "13825" uid: fca123a3-7b8d-4cc4-9b26-2d3664999306 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b6f09cc378 namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: 9bd1117c-4d80-4f61-b364-d3caef2d9bbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b6f371af02 namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: c4777724-c27f-4196-b18f-2e7f1f072e69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:44:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:47Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b6f3e675b1 namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 57ed705a-e3ec-440c-a1e5-7ddc3f99a4fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:44:52Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-21T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:44:52Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869b803445c36 namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: 9fd891a9-cbdb-41bf-a43a-d19b5c2fa067 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:45:10Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.208s (18.208s including waiting). Image size: 1306414327 bytes.' metadata: creationTimestamp: "2026-04-21T15:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:10Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869bc40982516 namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: 19b040b8-ffda-4491-9a90-42a3efeff3a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:45:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:10Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869bc443ebfdc namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 9b866852-88ff-4c25-854d-641abc04996f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:45:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:45:10Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869bc44b09dee namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 4811702c-914c-44db-ad63-2ef25efdb1bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-21T15:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:46:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:02Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869bc74b4409f namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 95a9ba13-8d17-4651-8b7c-a2b20a97058c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: afca5dff-afe5-41f5-94b9-2ef013483902 kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:26Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh.18a869ce1b5a85f5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 8931b6fc-73c4-45b1-be77-ede04df9b989 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 116efb1c-ea40-449c-bfd0-9962ce589755 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'Created pod: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5-xrhlh' metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5.18a869b6d11580d3 namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: f3cc4aef-33c3-44b0-b378-aec7bed9bd21 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 70830d78-7ba1-4a8a-af55-2ab74c539045 kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: Scaled up replica set isvc-xgboost-graph-raw-4bd9d-predictor-cb5d76c5 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-xgboost-graph-raw-4bd9d-predictor.18a869b6c69d76cb namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: 9bda3009-a940-4899-bbd3-1860ffb11f68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: a905fff4-fc2c-4060-be8b-a897ba4311bf kind: Event lastTimestamp: "2026-04-21T15:45:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:45:31Z" name: isvc-xgboost-graph-raw-4bd9d-predictor.18a869ba44c90c09 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 425b83d0-adf4-45fd-9f01-933a51e41c6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: a905fff4-fc2c-4060-be8b-a897ba4311bf kind: Event lastTimestamp: "2026-04-21T15:45:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:45:31Z" name: isvc-xgboost-graph-raw-4bd9d-predictor.18a869ba456df76d namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 7c9ed213-6868-48ee-b294-2e297936acab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: a905fff4-fc2c-4060-be8b-a897ba4311bf kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: isvc-xgboost-graph-raw-4bd9d-predictor.18a869c4c2e053f2 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: ae10f849-48af-4645-8b75-c5204f12b530 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4bd9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: a905fff4-fc2c-4060-be8b-a897ba4311bf kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: isvc-xgboost-graph-raw-4bd9d-predictor.18a869c4c3578306 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 06fa0405-0820-43b9-b220-808e1f3433b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 79b75206-ff5c-4bc5-be34-098705e5ecdb kind: Event lastTimestamp: "2026-04-21T15:46:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:46:12Z" name: isvc-xgboost-graph-raw-4bd9d.18a869b6c556a110 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 0037b12b-e027-4738-ba99-6179c1966544 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 79b75206-ff5c-4bc5-be34-098705e5ecdb kind: Event lastTimestamp: "2026-04-21T15:44:46Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-4bd9d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-4bd9d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:44:46Z" name: isvc-xgboost-graph-raw-4bd9d.18a869b6c8238f85 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: 4ae4b7c6-0659-4546-b38b-cdcd5be6fae1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 79b75206-ff5c-4bc5-be34-098705e5ecdb kind: Event lastTimestamp: "2026-04-21T15:46:12Z" message: InferenceService [isvc-xgboost-graph-raw-4bd9d] is Ready metadata: creationTimestamp: "2026-04-21T15:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:46:12Z" name: isvc-xgboost-graph-raw-4bd9d.18a869caad6e0e83 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 0f343626-8e03-43f9-9cd1-af14b60cc9af reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:46:26.829672Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:46:26Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869ce14bb98b6 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 4605b650-049b-42a1-aa31-82357c266037 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:46:27Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869ce2fc620dc namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: a2e57f45-134a-4547-8d88-d6c43fa31458 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869ce314a7444 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: e5b9656f-cf6e-430e-b8f3-fd4f53cfa16f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869ce3455656e namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 33802b06-8e1c-4558-9c96-9f8b83102536 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:27Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869ce34cafae6 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 44b29c67-0724-46f5-8cc7-09e20f2ab249 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869cf26d72bc3 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 8de5e0f7-82de-412c-b36d-0ca7ef64e055 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869cf2a772f04 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 1fd4b619-bb78-498b-9206-c69252e152d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:46:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:31Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869cf2b24b637 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 9905dbf4-6341-41f7-80e4-b107a2a4b48e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-21T15:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:47:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:47:23Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869cf63057cff namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 023c57fa-73ad-4585-8d7a-dfc33b61311b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0c39f490-81c3-4bac-9581-b765f2047827 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:17Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc.18a869e7c4fee777 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 576be731-f0fd-46ff-b87d-5824bb575ec6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99 namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: 2cc120c1-a658-46fd-877f-d5ce8343e255 kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99-2n4qc' metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:26Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99.18a869ce142105f0 namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 063dd753-0357-4e00-972f-2af99eb6800c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: db7d3cda-5cb3-407a-96e0-7eabedfb52a9 kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3991c-predictor-6d58c8fd99 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:26Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor.18a869ce116bf1e2 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: e5b22665-bc17-41a5-a5f4-526d8690c101 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: eff2a167-637d-432e-838e-7b6992c2901d kind: Event lastTimestamp: "2026-04-21T15:46:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:41Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor.18a869d19025ba2a namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: decb8ec4-fe5c-4367-a7bd-0eb98042a10f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: eff2a167-637d-432e-838e-7b6992c2901d kind: Event lastTimestamp: "2026-04-21T15:46:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:41Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor.18a869d1909dc703 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 13e23732-a4c7-45c9-8b03-afe1ace59996 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-21T15:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: eff2a167-637d-432e-838e-7b6992c2901d kind: Event lastTimestamp: "2026-04-21T15:47:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:47:56Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor.18a869d50f93e154 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7d4897d-de31-4f2e-ad56-c18ac4f93ea1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-21T15:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3991c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: eff2a167-637d-432e-838e-7b6992c2901d kind: Event lastTimestamp: "2026-04-21T15:47:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:47:56Z" name: isvc-xgboost-graph-raw-hpa-3991c-predictor.18a869d5100b9cce namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 9c41137b-b7e7-4b67-9ca1-4cab5c196c46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 0636d54e-6050-478e-8476-c53cee2601ee kind: Event lastTimestamp: "2026-04-21T15:47:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:47:33Z" name: isvc-xgboost-graph-raw-hpa-3991c.18a869ce1173d4bf namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 02048279-e143-43f9-b549-1e7420711bfb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: 0636d54e-6050-478e-8476-c53cee2601ee kind: Event lastTimestamp: "2026-04-21T15:47:33Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3991c] is Ready metadata: creationTimestamp: "2026-04-21T15:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:47:33Z" name: isvc-xgboost-graph-raw-hpa-3991c.18a869dd9c2dd9e5 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: c0692f4d-b2c5-42d8-a952-dbac7127fcd6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:48:17.095973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e7c11dd1fc namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 67fcdee9-d8b3-41a4-b33b-3de94b97c29d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e7c91f03ee namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 6e7fcd69-b734-4df3-a7cc-b9c9cc4b196c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e7ca89d06a namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: c1cb2bcd-8f98-4f20-9d69-3cc772c130b6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:48:18Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.116s (1.116s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-21T15:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:18Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e80d133bc6 namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 7c580e1a-3ce2-460b-9431-1d880e430db6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:48:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:18Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e81482adca namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 6e70b2d8-790a-4421-89b6-290ef9547da2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:48:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:18Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869e8150050c7 namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 8f409604-86c9-4d7c-9de5-4e964646e078 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 6cdb1f93-9c12-47ac-9a6f-61cb0bb8ae31 kind: Event lastTimestamp: "2026-04-21T15:49:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:49:52Z" name: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5.18a869fe0595d9b0 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 0aededba-b922-449e-8cdf-e548dc331166 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-61a06-predictor-cf9788b8b namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 8455a9a8-929c-4653-b081-1cea528d0e27 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: 'Created pod: message-dumper-raw-61a06-predictor-cf9788b8b-9nls5' metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06-predictor-cf9788b8b.18a869e7c0d19ce7 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: ecc626cf-a1e2-4385-92d1-26dfb48f27ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: f6bacd59-bc52-4792-8988-00932e2715fa kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: Scaled up replica set message-dumper-raw-61a06-predictor-cf9788b8b from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06-predictor.18a869e7bc6c2262 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 58118b99-f4eb-4909-8dc4-ad3df09618c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 6edcda0e-907a-42ff-aa4b-abbd5a70cf84 kind: Event lastTimestamp: "2026-04-21T15:48:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:32Z" name: message-dumper-raw-61a06-predictor.18a869eb3b3ba2b0 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: d5333767-c63f-41ce-bfec-2a63015fc372 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-61a06-predictor namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 6edcda0e-907a-42ff-aa4b-abbd5a70cf84 kind: Event lastTimestamp: "2026-04-21T15:48:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:32Z" name: message-dumper-raw-61a06-predictor.18a869eb3bddd025 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: e2affa26-482e-4c54-97cb-354f79ce9c1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: a3c2497d-4799-4154-9c8f-a825a7862783 kind: Event lastTimestamp: "2026-04-21T15:48:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:18Z" name: message-dumper-raw-61a06.18a869e7bbabc18a namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 2e8bc8b3-3331-4413-b000-def3a9d579c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: a3c2497d-4799-4154-9c8f-a825a7862783 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: 'Failed to update status for InferenceService "message-dumper-raw-61a06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-61a06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06.18a869e7bced8481 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: fb276176-1a5f-43fa-be65-c0e3540a4640 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: a3c2497d-4799-4154-9c8f-a825a7862783 kind: Event lastTimestamp: "2026-04-21T15:48:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-61a06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:17Z" name: message-dumper-raw-61a06.18a869e7c32e651c namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: c4820ba3-687e-4f67-8cae-d5c70b37999f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-61a06 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: a3c2497d-4799-4154-9c8f-a825a7862783 kind: Event lastTimestamp: "2026-04-21T15:48:18Z" message: InferenceService [message-dumper-raw-61a06] is Ready metadata: creationTimestamp: "2026-04-21T15:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:18Z" name: message-dumper-raw-61a06.18a869e82c27d183 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 18a7b78d-c07c-44fd-ba5b-df1fa5301df0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:46:16.534777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-4bd9d-b569d849-26rp9 to ip-10-0-133-158.ec2.internal metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cbaf1bef70 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: efcebd66-4f15-4001-a23a-48576f33f6e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cbc8106996 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 32c8a66a-b18e-4db8-9253-79fdc7728a18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:85893624ca442ac717d0ce5a9411858686c59f207eb581069a52d014f60caab4" metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cbc9d9c641 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 5bf12d9b-60fa-4554-92c7-d8e7a0f645a9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:85893624ca442ac717d0ce5a9411858686c59f207eb581069a52d014f60caab4" in 2.381s (2.381s including waiting). Image size: 216233529 bytes.' metadata: creationTimestamp: "2026-04-21T15:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:19Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cc57cfee1e namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 9b1118d3-ca51-414d-b08e-cf3636e3d8d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:19Z" message: 'Created container: model-chainer-raw-4bd9d' metadata: creationTimestamp: "2026-04-21T15:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:19Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cc5b1dae9f namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: f114a9da-209e-4b50-878f-66d6c6e878c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:19Z" message: Started container model-chainer-raw-4bd9d metadata: creationTimestamp: "2026-04-21T15:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:19Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869cc5b83f90e namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 40b313f6-df10-419e-b916-8a721dccfa82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:26Z" message: Stopping container model-chainer-raw-4bd9d metadata: creationTimestamp: "2026-04-21T15:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:26Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869ce061eb7ce namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: f123f274-bb22-4395-bb75-23ebce5f56df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-21T15:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4bd9d} kind: Pod name: model-chainer-raw-4bd9d-b569d849-26rp9 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 30869df4-7634-4299-ac1b-cbd8660110fc kind: Event lastTimestamp: "2026-04-21T15:46:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-21T15:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:46:55Z" name: model-chainer-raw-4bd9d-b569d849-26rp9.18a869ceef9788b1 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: e0aea0ec-1a5a-454e-b677-b5bb07f30365 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-4bd9d-b569d849 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 55f83229-5151-4431-8950-eb32579e633d kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: 'Created pod: model-chainer-raw-4bd9d-b569d849-26rp9' metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d-b569d849.18a869cbae6157e4 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 87de14ce-f99b-4759-a150-77218f2c04fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 95113905-1398-4347-b095-50bc154c68af kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: Scaled up replica set model-chainer-raw-4bd9d-b569d849 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d.18a869cbaca83785 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 9198bdb1-988e-46ca-874f-2793c26efe07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:46:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-4bd9d namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: c9fb0dab-e714-42a2-930c-10593eb0f5a8 kind: Event lastTimestamp: "2026-04-21T15:46:16Z" message: InferenceGraph [model-chainer-raw-4bd9d] is Ready metadata: creationTimestamp: "2026-04-21T15:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:46:16Z" name: model-chainer-raw-4bd9d.18a869cbb4e99bda namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 6ea496f4-e3d9-4ff4-ab62-167b5bae745f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:48:06.817973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3991c-bbfb6647c-5twzp to ip-10-0-133-158.ec2.internal metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e55c80018c namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 7e9ed746-3281-46c6-8eee-294097bae166 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:07Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:07Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:48:07Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e575659d1e namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 8176a8bf-0281-4878-92e9-4e67e8ed8d71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3991c} kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:85893624ca442ac717d0ce5a9411858686c59f207eb581069a52d014f60caab4" already present on machine metadata: creationTimestamp: "2026-04-21T15:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:07Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e576a45a3c namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 532a5ad2-3d03-4ac1-92f7-8ff2b37e5dea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3991c} kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:07Z" message: 'Created container: model-chainer-raw-hpa-3991c' metadata: creationTimestamp: "2026-04-21T15:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:07Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e579661358 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 5920a942-3c3f-4ada-bb02-318dad1257f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3991c} kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:07Z" message: Started container model-chainer-raw-hpa-3991c metadata: creationTimestamp: "2026-04-21T15:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:07Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e579c5c2f3 namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 79df0dd1-61fe-4ea4-a8be-4366d87441b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3991c} kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:16Z" message: Stopping container model-chainer-raw-hpa-3991c metadata: creationTimestamp: "2026-04-21T15:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:16Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e7b5044163 namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 39bd78e0-bb35-4943-8fb7-5a2041065cd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-21T15:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3991c} kind: Pod name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3325cfb3-5281-40b4-a5c9-178e451799fc kind: Event lastTimestamp: "2026-04-21T15:48:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-21T15:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:48:43Z" name: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp.18a869e825de95f5 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 22854917-bc5b-48cc-878e-d7f3f1a0e995 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-158.ec2.internal source: component: kubelet host: ip-10-0-133-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3991c-bbfb6647c namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 68a587d4-1ef4-4098-ace5-425e3613ab66 kind: Event lastTimestamp: "2026-04-21T15:48:06Z" message: 'Created pod: model-chainer-raw-hpa-3991c-bbfb6647c-5twzp' metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c-bbfb6647c.18a869e55c187631 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: e192138b-f3aa-4680-90d7-8e723df10d45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 8a191085-b613-43cb-8490-48d2ed69239f kind: Event lastTimestamp: "2026-04-21T15:48:06Z" message: Scaled up replica set model-chainer-raw-hpa-3991c-bbfb6647c from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c.18a869e55b253302 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: fe28195d-cf1e-4481-ae9a-0651a30d21dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 6cbf923a-eaf5-4f9e-9e72-4c646eccc436 kind: Event lastTimestamp: "2026-04-21T15:48:06Z" message: InferenceGraph [model-chainer-raw-hpa-3991c] is Ready metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c.18a869e56050fc9e namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: e6dff1bc-a1cc-4569-b2b1-ad8607f9a3b8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 6cbf923a-eaf5-4f9e-9e72-4c646eccc436 kind: Event lastTimestamp: "2026-04-21T15:48:06Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-3991c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-3991c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c.18a869e562ceeca8 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 1bd05a1a-ae16-4c2d-b815-e206c663f1df reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:48:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3991c namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 6cbf923a-eaf5-4f9e-9e72-4c646eccc436 kind: Event lastTimestamp: "2026-04-21T15:48:06Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-3991c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:48:06Z" name: model-chainer-raw-hpa-3991c.18a869e562cf3bdf namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: c66661a4-e195-4da5-9639-b4dffc3ccfa1 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:54:21.366145Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18840" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3c914f9d98 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: c200c69b-88f3-47b1-ba5b-a03b69fd1535 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3caacf5ee4 namespace: kserve-ci-e2e-test resourceVersion: "18862" uid: 7eee755d-75fd-4220-a00c-40e35a957019 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3cac3b1f45 namespace: kserve-ci-e2e-test resourceVersion: "18864" uid: ca7f0d6d-3d70-45ec-abac-986b47d23860 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3cafada121 namespace: kserve-ci-e2e-test resourceVersion: "18868" uid: c15833a2-4759-4679-8dfd-81dbc1c30533 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3cb029ae29 namespace: kserve-ci-e2e-test resourceVersion: "18869" uid: 3fbdb09f-f871-4a42-bf05-9fbf6f584d8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:26Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:26Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3daf5b559e namespace: kserve-ci-e2e-test resourceVersion: "18916" uid: 3828999e-2c60-4a60-abdd-b316eeae5cae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:26Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3db2aa9fbf namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: 96042564-f01e-4dc4-809f-1be55b77b0d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:54:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:54:26Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3db322aeb3 namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 866d4284-cf2c-44c5-b305-c01d39ff0756 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:54:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:55:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:28Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a3deb4fd08d namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: 5ebe2b5c-c9dc-4156-a614-85a8de79bfd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 8f822518-dd23-4cc3-b26b-556414043e59 kind: Event lastTimestamp: "2026-04-21T15:55:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:41Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr.18a86a4f4430833c namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: dc4892e4-16e8-4eb1-8b9f-1345f8aaf32e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-e5dc9-predictor-b76fc85f7 namespace: kserve-ci-e2e-test resourceVersion: "18832" uid: 2be7c1a7-5fbe-4f16-b211-652d88fc2d4c kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: 'Created pod: raw-sklearn-e5dc9-predictor-b76fc85f7-5tnlr' metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor-b76fc85f7.18a86a3c90e9313b namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 0c161135-c960-40a6-9eb6-e70f891f1850 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-e5dc9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: deaa4e6a-60f7-4cac-87fc-bb7bf473a40c kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: Scaled up replica set raw-sklearn-e5dc9-predictor-b76fc85f7 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9-predictor.18a86a3c8e4fa612 namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 27dd60db-e392-4669-82fe-e59989fea998 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e5dc9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 7a7f408e-8ad8-4ae4-975d-4efbab652e01 kind: Event lastTimestamp: "2026-04-21T15:54:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:51Z" name: raw-sklearn-e5dc9-predictor.18a86a400cfce4ad namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 98ff1382-7797-4d75-b697-f8bdcabc61ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e5dc9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 7a7f408e-8ad8-4ae4-975d-4efbab652e01 kind: Event lastTimestamp: "2026-04-21T15:54:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:54:51Z" name: raw-sklearn-e5dc9-predictor.18a86a400d7c74b5 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 65f256c2-abe3-4cbe-aad3-3708448f8167 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e5dc9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: 7a7f408e-8ad8-4ae4-975d-4efbab652e01 kind: Event lastTimestamp: "2026-04-21T15:55:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:55:36Z" name: raw-sklearn-e5dc9-predictor.18a86a470c655d15 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 14847b01-6c6e-4512-9ec1-337c9b823d73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-21T15:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e5dc9-predictor namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: 7a7f408e-8ad8-4ae4-975d-4efbab652e01 kind: Event lastTimestamp: "2026-04-21T15:55:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:55:36Z" name: raw-sklearn-e5dc9-predictor.18a86a470d0612a1 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: fbdbf2c8-7643-4cc5-bfab-30cdf2bab3db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e5dc9 namespace: kserve-ci-e2e-test resourceVersion: "18821" uid: e3cfb5f1-a2b0-4e7c-bc8d-62784902cc20 kind: Event lastTimestamp: "2026-04-21T15:55:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:55:36Z" name: raw-sklearn-e5dc9.18a86a3c8e283345 namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: 88e1f8c8-0df4-4d02-bfba-36ec5d06f785 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e5dc9 namespace: kserve-ci-e2e-test resourceVersion: "18821" uid: e3cfb5f1-a2b0-4e7c-bc8d-62784902cc20 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: 'Failed to update status for InferenceService "raw-sklearn-e5dc9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-e5dc9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9.18a86a3c8e81f35d namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 0b3b8dd6-7d51-4950-b0dc-0e1a932dc2fe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:54:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e5dc9 namespace: kserve-ci-e2e-test resourceVersion: "18838" uid: e3cfb5f1-a2b0-4e7c-bc8d-62784902cc20 kind: Event lastTimestamp: "2026-04-21T15:54:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-e5dc9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:54:21Z" name: raw-sklearn-e5dc9.18a86a3c94b38ee4 namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 93eb7381-e2a1-499b-a6bf-76db42593867 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e5dc9 namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: e3cfb5f1-a2b0-4e7c-bc8d-62784902cc20 kind: Event lastTimestamp: "2026-04-21T15:55:36Z" message: InferenceService [raw-sklearn-e5dc9] is Ready metadata: creationTimestamp: "2026-04-21T15:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:55:36Z" name: raw-sklearn-e5dc9.18a86a4e26614331 namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: fdd618f9-de90-461e-a294-68dcedf6b103 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-21T15:55:41.623899Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 to ip-10-0-136-162.ec2.internal metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-21T15:55:41Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a4f410ba685 namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: 36819285-b850-41c3-9eff-51662111b849 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-678d6985f7-v49pq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:42Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:42Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-21T15:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-21T15:55:42Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a4f5a87b64d namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: c435b72d-0253-4100-b4f2-f20dc97b470c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6917b7549dba28168fe9ddab83ad49bb5876a127ac47aab2659411acc81a41e6" already present on machine metadata: creationTimestamp: "2026-04-21T15:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:42Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a4f5c021857 namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: b46b3181-867b-4ebf-a04e-63b505185e9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-21T15:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:42Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a4f5f2a7866 namespace: kserve-ci-e2e-test resourceVersion: "19521" uid: 980fa8d1-2254-43b2-9a75-f8d83cd7c9a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-21T15:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:42Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a4f5fb0901c namespace: kserve-ci-e2e-test resourceVersion: "19522" uid: 775ec0fa-c6f4-4627-9ff6-c86062725d02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1428" already present on machine metadata: creationTimestamp: "2026-04-21T15:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:46Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a5060c2eaf6 namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 3a440f11-2589-469e-8d87-7f8cc4f954b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-21T15:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:46Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a5064a32f1b namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: 1c40c2da-8ba0-45b3-9f72-bdd56e775b9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:55:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-21T15:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:55:46Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a50652030b5 namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: 837d7f60-2594-48bc-8a1c-9dde7dfda48c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-21T15:55:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:56:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-21T15:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:56:48Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a509d338485 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 3f7c9da7-f1ac-4db5-b1a5-8e9490cfaa66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:57:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: a315b444-99ef-4eae-b5b8-9ed2e0a95c56 kind: Event lastTimestamp: "2026-04-21T15:57:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-21T15:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-21T15:57:01Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8.18a86a61f1034352 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 0135bb26-eddf-45c4-94df-b00a57fe6b10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-162.ec2.internal source: component: kubelet host: ip-10-0-136-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: ad3980c4-24ae-42af-9b86-76f9ba9e5abd kind: Event lastTimestamp: "2026-04-21T15:55:41Z" message: 'Created pod: raw-sklearn-runtime-06bac-predictor-6f4db74cd7-hc5h8' metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:55:41Z" name: raw-sklearn-runtime-06bac-predictor-6f4db74cd7.18a86a4f406f822d namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: e5f879a8-c49c-4449-9e79-d05821b719c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-06bac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 6171dc60-ea64-4b21-a4a9-a1b463e10439 kind: Event lastTimestamp: "2026-04-21T15:55:41Z" message: Scaled up replica set raw-sklearn-runtime-06bac-predictor-6f4db74cd7 from 0 to 1 metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:55:41Z" name: raw-sklearn-runtime-06bac-predictor.18a86a4f3dea861b namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 9c572f08-121e-4213-aa05-0643cd623c6f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-06bac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: f3b950ff-d837-483c-bda9-9f8927e519cc kind: Event lastTimestamp: "2026-04-21T15:56:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:56:41Z" name: raw-sklearn-runtime-06bac-predictor.18a86a52bd6ced86 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 0ea6b71d-a908-4156-9f88-edde9d868e78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-21T15:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-06bac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: f3b950ff-d837-483c-bda9-9f8927e519cc kind: Event lastTimestamp: "2026-04-21T15:56:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-21T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:56:41Z" name: raw-sklearn-runtime-06bac-predictor.18a86a52be23778f namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 3f7d1b5b-a6ec-4199-a7e5-4ab393bc17b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-06bac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: f3b950ff-d837-483c-bda9-9f8927e519cc kind: Event lastTimestamp: "2026-04-21T15:56:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:56:56Z" name: raw-sklearn-runtime-06bac-predictor.18a86a60bad37e55 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 2919b206-343d-49ca-b664-e11aab953ef6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-06bac-predictor namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: f3b950ff-d837-483c-bda9-9f8927e519cc kind: Event lastTimestamp: "2026-04-21T15:56:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-21T15:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-21T15:56:56Z" name: raw-sklearn-runtime-06bac-predictor.18a86a60bb47b131 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: e681f053-4d0c-4bc0-a730-42b054989355 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-21T15:55:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-06bac namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 7aa72aea-062d-48ac-b97f-3a2d65937f0a kind: Event lastTimestamp: "2026-04-21T15:56:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:56:57Z" name: raw-sklearn-runtime-06bac.18a86a4f3db3a732 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 6174a80d-a3d7-4caf-88f2-0506aac7cc9b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:55:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-06bac namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 7aa72aea-062d-48ac-b97f-3a2d65937f0a kind: Event lastTimestamp: "2026-04-21T15:55:41Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-06bac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-06bac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-21T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:55:41Z" name: raw-sklearn-runtime-06bac.18a86a4f3e2935e3 namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 27c11fae-e30b-4010-a0fc-e6eb54a7492a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-21T15:56:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-06bac namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 7aa72aea-062d-48ac-b97f-3a2d65937f0a kind: Event lastTimestamp: "2026-04-21T15:56:57Z" message: InferenceService [raw-sklearn-runtime-06bac] is Ready metadata: creationTimestamp: "2026-04-21T15:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-21T15:56:57Z" name: raw-sklearn-runtime-06bac.18a86a61020a1e47 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 2703c768-3f37-4cea-a6a5-71735e726be9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20337"