--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:24:01.104063Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f41b6a0eae namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: dbecc410-ba57-4193-a5f5-613a28d30d15 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f43584d396 namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: ebd37992-f3f0-48e5-b5f8-a3033d899acb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: "2026-04-16T16:24:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:08Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f4371f3e88 namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: d4f50598-7f7d-475a-8582-fa8e6ad71d14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: "2026-04-16T16:24:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:08Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f43a3a6f80 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: e5dfbdc1-436a-45aa-b589-323e968e0a53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: "2026-04-16T16:24:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:08Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f43aad276b namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: 5ce4b61f-f590-486b-8cf3-85e2af10f795 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 8e540668-c921-499a-b737-e8b918f2be29 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:11Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv.18a6e2f6714285f3 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: cb660976-b967-4763-b8ca-93acdb5e00b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-b43ebd-predictor-84bd96d595 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: b76276f8-57ac-44b0-91fb-88a05df0b8b4 kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: 'Created pod: isvc-init-fail-b43ebd-predictor-84bd96d595-89tkv' metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd-predictor-84bd96d595.18a6e2f41b0ebbb4 namespace: kserve-ci-e2e-test resourceVersion: "19544" uid: 4b3e1107-951e-4ef7-949d-dac4afa6fbdd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-b43ebd-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: 5fb47fb0-80f4-41b2-9f0e-7a1d279e8638 kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: Scaled up replica set isvc-init-fail-b43ebd-predictor-84bd96d595 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd-predictor.18a6e2f4186aadf4 namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: df66abb4-383a-4244-8a93-e19606d30760 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b43ebd namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a9bf76d3-8334-4b9b-8902-6bb95cf4afbf kind: Event lastTimestamp: "2026-04-16T16:24:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:08Z" name: isvc-init-fail-b43ebd.18a6e2f4184aa1ee namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: e70380b1-5650-4ae3-b8af-47b4756a47c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b43ebd namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a9bf76d3-8334-4b9b-8902-6bb95cf4afbf kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: 'Failed to update status for InferenceService "isvc-init-fail-b43ebd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b43ebd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd.18a6e2f418bb319f namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: 398a88d3-a2fb-4ab5-bf1e-caf70f47bfe0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b43ebd namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a9bf76d3-8334-4b9b-8902-6bb95cf4afbf kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b43ebd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:01Z" name: isvc-init-fail-b43ebd.18a6e2f418bb8bc1 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 0b8bd57d-b136-4968-a099-f91ef84f8e05 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:18:27.322866Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a664812a5d namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 0abb3d7e-9d87-444a-b4bb-5811f242c5f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a67d8088dc namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 6b00d141-f2f4-4ce8-8472-d86494f91e01 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a67eebed6b namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: e17f04f9-119e-4ee8-a6d9-9bf424f216d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a681e56f3f namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: d2007dc6-7c36-4295-ace7-4e1409e76f07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a682542539 namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 940e31df-60d1-4fad-b4cd-3b777122c91a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a78680f6e1 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: b6c0ea53-b46d-4af7-90b0-87d2a4e1d2c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a789512771 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 3de31852-564e-4b12-a027-63c70daa7f24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a789f0270c namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: f68df4d3-d899-43ad-b4d6-1d2016833f83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" already present on machine metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a789fcc32c namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 18314595-0af1-417a-87b9-1417d8d45e5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a78cfa5436 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 64baf1ac-63d8-4930-aa4e-aeb689036b46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:18:32Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:32Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a78d6eb20c namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: d9afcbc0-235f-4da4-9085-6227acfe09b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T16:18:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:20:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:20:14Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a7c27f9789 namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: 65b3a7be-5824-4e86-b938-1f415dab7fc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T16:18:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:20:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:20:14Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2a7c288d569 namespace: kserve-ci-e2e-test resourceVersion: "17733" uid: 0a35a9d7-e1b7-43ec-aa45-6c34bde38df6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:52Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2ba3afb11de namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 1a4bc0b6-241d-4ea7-9c29-37932cf0ea23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: e7bcf803-b1ee-4125-94ab-1012599c0fa0 kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:52Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b.18a6e2ba3afc4f81 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: b885acc1-3666-4d3a-8970-1404404ab732 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-74f70-predictor-86f49fcb48 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 0139c4da-2d3c-4885-875c-93348db64156 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: 'Created pod: isvc-logger-raw-74f70-predictor-86f49fcb48-hxk5b' metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor-86f49fcb48.18a6e2a663e2d0af namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: 4e8c0450-a85e-431e-a988-1520ae4e03f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: 33b7fa23-b5a6-4578-90f4-d52ca7e68d69 kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: Scaled up replica set isvc-logger-raw-74f70-predictor-86f49fcb48 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70-predictor.18a6e2a66076cd93 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 06f4aed5-a761-4cad-b098-25441eedf9c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: 58161b0a-368f-4237-85cb-60a629eb3521 kind: Event lastTimestamp: "2026-04-16T16:18:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:57Z" name: isvc-logger-raw-74f70-predictor.18a6e2a9def4822b namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: ad178314-9770-4e39-86a1-e5c8f6cab9b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: 58161b0a-368f-4237-85cb-60a629eb3521 kind: Event lastTimestamp: "2026-04-16T16:18:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:57Z" name: isvc-logger-raw-74f70-predictor.18a6e2a9df79b4ad namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: fabe7693-505c-4222-866c-b3068d047f8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:19:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: 58161b0a-368f-4237-85cb-60a629eb3521 kind: Event lastTimestamp: "2026-04-16T16:19:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:19:42Z" name: isvc-logger-raw-74f70-predictor.18a6e2b0dd4d1304 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 06d6add3-dae6-4e61-9106-522fd125cf20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:19:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: 58161b0a-368f-4237-85cb-60a629eb3521 kind: Event lastTimestamp: "2026-04-16T16:19:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:19:42Z" name: isvc-logger-raw-74f70-predictor.18a6e2b0ddb0b421 namespace: kserve-ci-e2e-test resourceVersion: "17400" uid: 3056ddcb-140f-4eb1-bead-508fe020aabe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 6dd9a887-a5f4-436e-a27c-2d503fbe418e kind: Event lastTimestamp: "2026-04-16T16:19:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:19:44Z" name: isvc-logger-raw-74f70.18a6e2a6605687dc namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 78c4b694-66b5-453c-8d48-b3642a7d50e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 6dd9a887-a5f4-436e-a27c-2d503fbe418e kind: Event lastTimestamp: "2026-04-16T16:18:27Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-74f70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-74f70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:18:27Z" name: isvc-logger-raw-74f70.18a6e2a660c3b3b8 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: b50a9f51-e776-4488-9e2d-474084b7e6c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "17425" uid: 6dd9a887-a5f4-436e-a27c-2d503fbe418e kind: Event lastTimestamp: "2026-04-16T16:19:44Z" message: InferenceService [isvc-logger-raw-74f70] is Ready metadata: creationTimestamp: "2026-04-16T16:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:19:44Z" name: isvc-logger-raw-74f70.18a6e2b84f389e27 namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: dbaac37d-f9f9-4998-8306-d9a637005d73 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:22:22.789755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18679" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-89be61-predictor-789bf94c5d-99btm to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:22:22Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dd376ccd45 namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: 5c885a4f-55a7-4d59-bec1-30512e40e8c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:23Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:22:23Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dd50a831ec namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: dcb0f5b0-8f42-479e-a6c7-c2a6bf554e8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:23Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:23Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dd51f0cd3b namespace: kserve-ci-e2e-test resourceVersion: "18697" uid: 3cc34311-c8a9-46df-a20b-23792ae56214 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:23Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dd54ebf9e8 namespace: kserve-ci-e2e-test resourceVersion: "18699" uid: c86a4653-b91a-44c1-9823-aa546437334b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:23Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dd5564d5dc namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: 37b14d21-ccc8-4f76-8d87-f50bb9a87c53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:27Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:27Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2de69e99ea8 namespace: kserve-ci-e2e-test resourceVersion: "18738" uid: 8108ae21-4b24-423a-bfd3-82a6ea8a51d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:27Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2de6d32da93 namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: 4bb045a7-d7f2-4402-901f-d0754eb2ae49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:22:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:27Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2de6db4380e namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: 78f06770-d814-42fb-940c-c28b43035a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:23:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:23:29Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2dea5d0ab82 namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: cf7404b1-3bb2-4c31-8151-17482114fd3c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-89be61-predictor-789bf94c5d-99btm namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: 4ee006b4-beab-4d69-94e9-eccc23549281 kind: Event lastTimestamp: "2026-04-16T16:24:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:01Z" name: isvc-primary-89be61-predictor-789bf94c5d-99btm.18a6e2f418cd5d08 namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 14dac139-d3ea-4e5a-bc24-68ab8e1d1d90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-89be61-predictor-789bf94c5d namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: 704e408b-0f9e-4081-b3ab-e3588bcbc12d kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: 'Created pod: isvc-primary-89be61-predictor-789bf94c5d-99btm' metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-primary-89be61-predictor-789bf94c5d.18a6e2dd3724dcad namespace: kserve-ci-e2e-test resourceVersion: "18682" uid: 81040a6e-7e7b-4226-bf6e-89c3c1efb16e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "18666" uid: 1bc1fc89-65b8-4a83-8346-ddb1bc8e691e kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: Scaled up replica set isvc-primary-89be61-predictor-789bf94c5d from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-primary-89be61-predictor.18a6e2dd32f0fe39 namespace: kserve-ci-e2e-test resourceVersion: "18672" uid: 8ec93142-145b-464c-84b0-bace074ac33e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:22:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: ad191429-2786-4d86-b38a-2ed7990793ab kind: Event lastTimestamp: "2026-04-16T16:22:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:52Z" name: isvc-primary-89be61-predictor.18a6e2e0b17ec4b6 namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: 2b750c36-ea4d-44b9-92c8-bb1c1af766ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:22:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: ad191429-2786-4d86-b38a-2ed7990793ab kind: Event lastTimestamp: "2026-04-16T16:22:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:52Z" name: isvc-primary-89be61-predictor.18a6e2e0b1f57997 namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 8810abe0-781d-4bba-a6d7-6cd7f2e6abe4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:23:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "18819" uid: ad191429-2786-4d86-b38a-2ed7990793ab kind: Event lastTimestamp: "2026-04-16T16:23:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:52Z" name: isvc-primary-89be61-predictor.18a6e2e7b0538b82 namespace: kserve-ci-e2e-test resourceVersion: "19420" uid: 8c6bfc79-5d16-490f-8762-247cdac4da5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:23:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "18819" uid: ad191429-2786-4d86-b38a-2ed7990793ab kind: Event lastTimestamp: "2026-04-16T16:23:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:52Z" name: isvc-primary-89be61-predictor.18a6e2e7b0c54ae3 namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: d69e160a-f5ad-4435-8d31-120bec3e93bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "18644" uid: d68ecb66-15d6-46e9-a017-4033065e4407 kind: Event lastTimestamp: "2026-04-16T16:23:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:23:40Z" name: isvc-primary-89be61.18a6e2dd32c1ba19 namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: da4fad8a-3933-4d09-9214-0d30aa7a3f97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "18644" uid: d68ecb66-15d6-46e9-a017-4033065e4407 kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: 'Failed to update status for InferenceService "isvc-primary-89be61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-89be61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-primary-89be61.18a6e2dd33412917 namespace: kserve-ci-e2e-test resourceVersion: "18674" uid: 985e73f1-fd9c-4358-8132-753c46f03705 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "18644" uid: d68ecb66-15d6-46e9-a017-4033065e4407 kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-89be61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-primary-89be61.18a6e2dd3341703c namespace: kserve-ci-e2e-test resourceVersion: "18675" uid: 06110fd0-89f1-43dc-b5d0-07c0b0b20d4b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: d68ecb66-15d6-46e9-a017-4033065e4407 kind: Event lastTimestamp: "2026-04-16T16:23:40Z" message: InferenceService [isvc-primary-89be61] is Ready metadata: creationTimestamp: "2026-04-16T16:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:23:40Z" name: isvc-primary-89be61.18a6e2ef31cd89b2 namespace: kserve-ci-e2e-test resourceVersion: "19272" uid: b30f4d3f-17ba-4259-93fd-db1630afc7a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:13:11.414567Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25cd6e71998 namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 723061f7-4481-4156-bfd3-7ea2225601e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25cf00576b9 namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 8476b60b-00dd-4cf9-95c7-6aaf4e6bf257 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25cf14084a4 namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: eb2638ca-ac03-4240-ad0f-f595f9c75cf6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" in 2.991s (2.991s including waiting). Image size: 299659693 bytes.' metadata: creationTimestamp: "2026-04-16T16:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:14Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25da38d24f3 namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: a24d36c7-ea75-4567-a6d1-d1c4862beef9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:14Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25da6d62b3b namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: 091cd49a-c72c-432e-b8cf-621f22d32c4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:14Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25da7401856 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: 2294d1cc-6dfb-431e-8056-edacb4a1c564 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:19Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" metadata: creationTimestamp: "2026-04-16T16:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:19Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e25ea2dc7cb5 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 7c9c326e-5837-407b-8fab-41fe247d78e1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" in 13.112s (13.112s including waiting). Image size: 1562961329 bytes.' metadata: creationTimestamp: "2026-04-16T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:32Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e261b06b1980 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: f291a606-959c-43d8-b08e-d85ffe29a11d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:32Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e261b362d171 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: af0eadd7-ff49-4b31-a46d-a066ac71b7ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:32Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e261b3c98ad6 namespace: kserve-ci-e2e-test resourceVersion: "14122" uid: 02ce58aa-6b56-4daf-9d14-022c3941aa56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:32Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" metadata: creationTimestamp: "2026-04-16T16:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:32Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e261b3d597d9 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: ba412fba-25a1-432b-a2d0-ac2b1175bdbf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9" in 2.37s (2.37s including waiting). Image size: 237894735 bytes.' metadata: creationTimestamp: "2026-04-16T16:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:34Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e262411dfaed namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: 09ed6ebd-041b-4610-af14-533b7d842e9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-16T16:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:34Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e26243e872b5 namespace: kserve-ci-e2e-test resourceVersion: "14140" uid: 070d35ec-1cae-49ae-816d-fe925e893f3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:13:34Z" message: Started container agent metadata: creationTimestamp: "2026-04-16T16:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:13:34Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e262444fb2ad namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: e9400bdc-1397-47af-abba-b8e6e4e6cec8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:15:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:17Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e26260caa834 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 1d0c0267-0484-4146-9380-ece45aebc296 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-16T16:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:15:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:17Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e2629c8a38d8 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: d555ebf7-0455-489b-bfa1-f4105f14edaa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e275549e6f15 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 123ad425-4752-486f-bded-c1f3403418d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e275549faed0 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 61be5bd5-64bd-4327-a052-5af277d81d43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:15:28Z" message: 'Readiness probe failed: Get "http://10.133.0.22:9081/": context deadline exceeded (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-04-16T16:15:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:28Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e27cb0714a78 namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 78997466-b88c-4731-b600-02bc12c0fe9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: b1676cda-2a69-4758-a9ca-a7dd11580d8d kind: Event lastTimestamp: "2026-04-16T16:15:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: i/o timeout' metadata: creationTimestamp: "2026-04-16T16:15:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:28Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8.18a6e27cb071a4fd namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 95841be9-43e8-4ff5-bd03-2a31943831eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: bf0e01db-f0fc-490f-b34b-2e690064726a kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: 'Created pod: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d-rxgg8' metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d.18a6e25cd643656d namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: a1db0971-ad19-4c0b-8179-c3f42c85c04c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-70fe9-predictor namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: a14446e4-ea42-4d37-844d-cfd89090ba15 kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: Scaled up replica set isvc-raw-sklearn-batcher-70fe9-predictor-667cd4c84d from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9-predictor.18a6e25cce0c401d namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: d98e32c6-47d7-4d4e-9cc8-9eb90021fbce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-70fe9-predictor namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: 3b1ac4f7-4078-42ab-96d6-508abbe92c25 kind: Event lastTimestamp: "2026-04-16T16:13:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:13:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor.18a6e260537781d0 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: a79515f1-18df-47fb-af8c-91080e4a59f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-70fe9-predictor namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: 3b1ac4f7-4078-42ab-96d6-508abbe92c25 kind: Event lastTimestamp: "2026-04-16T16:13:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:13:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor.18a6e26053df47d2 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: a3acce3c-884c-4b69-b06c-1b9e1e25655c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:14:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-70fe9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 3b1ac4f7-4078-42ab-96d6-508abbe92c25 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor.18a6e26ad0cf3cc4 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: c1f0396c-e2bf-4832-9b12-0c3ab260ffa3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:14:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-70fe9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 3b1ac4f7-4078-42ab-96d6-508abbe92c25 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-raw-sklearn-batcher-70fe9-predictor.18a6e26ad142b209 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 3958d75e-a9e1-4535-80fa-e87f8c888f98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-70fe9 namespace: kserve-ci-e2e-test resourceVersion: "13920" uid: b826018d-8b98-4c14-ab7b-2bdfae1e1913 kind: Event lastTimestamp: "2026-04-16T16:14:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:14:47Z" name: isvc-raw-sklearn-batcher-70fe9.18a6e25cd41f2c65 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 8780e11b-28a4-4fde-bf7f-c864b8cb8dd5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-70fe9 namespace: kserve-ci-e2e-test resourceVersion: "13920" uid: b826018d-8b98-4c14-ab7b-2bdfae1e1913 kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-70fe9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-70fe9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9.18a6e25cd46e3efd namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: c8a5c979-bb2c-47bb-b715-0e0a90034c3d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-70fe9 namespace: kserve-ci-e2e-test resourceVersion: "13920" uid: b826018d-8b98-4c14-ab7b-2bdfae1e1913 kind: Event lastTimestamp: "2026-04-16T16:13:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-70fe9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:13:11Z" name: isvc-raw-sklearn-batcher-70fe9.18a6e25cd46ea80f namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: a0b0f214-5980-4975-a62d-ab6a0397dd63 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-70fe9 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: b826018d-8b98-4c14-ab7b-2bdfae1e1913 kind: Event lastTimestamp: "2026-04-16T16:14:47Z" message: InferenceService [isvc-raw-sklearn-batcher-70fe9] is Ready metadata: creationTimestamp: "2026-04-16T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:14:47Z" name: isvc-raw-sklearn-batcher-70fe9.18a6e27330e58719 namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 2db92c01-14d9-43c3-9b40-25e4b10028c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:23:42.931028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:23:42Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2efe0378cc9 namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: bd70a886-ff5f-4e0d-b79b-079741f8cbdb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: "2026-04-16T16:23:43Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:23:43Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2eff94eb3bb namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 33b0912b-9999-4c98-bbdb-961d4df49ba4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:23:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: "2026-04-16T16:23:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:23:49Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2effa9aca8f namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 6fd92d63-82fe-4f58-b8fc-0d283476625f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:23:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: "2026-04-16T16:23:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:23:49Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2effd8fa67b namespace: kserve-ci-e2e-test resourceVersion: "19393" uid: 67fb5af0-2f38-4aaf-891d-01f847562f25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:23:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: "2026-04-16T16:23:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:23:49Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2effe022085 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: ed724024-8532-4067-bf2c-c685b36a9dda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 4e44d3b1-7732-4fc5-81d9-17269abf124a kind: Event lastTimestamp: "2026-04-16T16:23:56Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp_kserve-ci-e2e-test(4e44d3b1-7732-4fc5-81d9-17269abf124a) metadata: creationTimestamp: "2026-04-16T16:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:23:56Z" name: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp.18a6e2f2f8ef5151 namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: b66050c8-a514-447f-90d5-a1c55733fcee reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-89be61-predictor-5bdc697fb7 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: f55824b7-49eb-49b9-8e03-5a60743ecd10 kind: Event lastTimestamp: "2026-04-16T16:23:42Z" message: 'Created pod: isvc-secondary-89be61-predictor-5bdc697fb7-9ndcp' metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:42Z" name: isvc-secondary-89be61-predictor-5bdc697fb7.18a6e2efdfc9c3bf namespace: kserve-ci-e2e-test resourceVersion: "19320" uid: 0a4bdb02-3f6d-4265-b148-5cf2d8aa62d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: e6576d1f-552b-4b9a-9f4d-60005b9e04f2 kind: Event lastTimestamp: "2026-04-16T16:23:42Z" message: Scaled up replica set isvc-secondary-89be61-predictor-5bdc697fb7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:42Z" name: isvc-secondary-89be61-predictor.18a6e2efdcdf958d namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 43d13759-e8e2-4bd8-ab6f-3d273b39ad94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 238cff9f-9e87-48c5-8a63-3d8f105ce7fd kind: Event lastTimestamp: "2026-04-16T16:23:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:57Z" name: isvc-secondary-89be61-predictor.18a6e2f35b4742e6 namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: bfac5e74-4a1d-41d5-804b-9389a5b1e43d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:23:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-89be61-predictor namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 238cff9f-9e87-48c5-8a63-3d8f105ce7fd kind: Event lastTimestamp: "2026-04-16T16:23:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:23:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:23:57Z" name: isvc-secondary-89be61-predictor.18a6e2f35ba76c12 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: d7effef5-3421-4831-9a3e-b288f72ed6dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T16:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 5f3f849d-3bc4-4b4a-ab2c-9954fb6d521b kind: Event lastTimestamp: "2026-04-16T16:24:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:00Z" name: isvc-secondary-89be61.18a6e2efdcb5fd9a namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 37673008-45ba-4910-b1cf-520af36a3f53 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 5f3f849d-3bc4-4b4a-ab2c-9954fb6d521b kind: Event lastTimestamp: "2026-04-16T16:23:42Z" message: 'Failed to update status for InferenceService "isvc-secondary-89be61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-89be61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:23:42Z" name: isvc-secondary-89be61.18a6e2efdd1ee4a7 namespace: kserve-ci-e2e-test resourceVersion: "19329" uid: b6fe4012-5e0b-45fc-ad93-cef87a3ffe91 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-89be61 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 5f3f849d-3bc4-4b4a-ab2c-9954fb6d521b kind: Event lastTimestamp: "2026-04-16T16:23:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-89be61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:23:42Z" name: isvc-secondary-89be61.18a6e2efdd1f30e4 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: daa1ff61-d3ec-42a5-855f-2a11bde5bda0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:14:56.713686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:14:56Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e2755b38206d namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 7d8d7a0f-9994-4fb3-b523-18ec0aa7f130 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:14:57Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e27574ff1f8d namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 5c36068f-3d11-461c-bfac-1d66795fcf22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e275765f3e0f namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: b70a0a8e-f906-44a7-9c90-28aaa22a91fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e27579865b8e namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 388b7c6c-15e2-4b27-8699-547f3bd4ecc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e27579ecc29b namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: a69793ae-48f8-410c-87b6-737a5860f94f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:15:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:01Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e276773df52d namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 79c570be-789c-49cb-b496-fcb8e9bb0220 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:15:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:01Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e2767a0a5599 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: f9e2cdd6-883e-4530-a1d2-d31efd9b2ca8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:15:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:01Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e2767a78267d namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: b2d3d355-7d94-4a7b-bcf2-d7a618d3f86f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:16:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:03Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e276b375b75d namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 16a96169-84e7-45ac-a25c-8a3447777081 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 8a520d0f-1263-46a8-901b-93faf79b59de kind: Event lastTimestamp: "2026-04-16T16:16:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:36Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g.18a6e28cb1f62631 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: fb7b7eac-c2e9-4882-92e9-84cea6a109c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 58062417-254d-43e4-9be4-4deb51f3f987 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'Created pod: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8-42v2g' metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-sklearn-graph-raw-3b643-predictor-584c5864d8.18a6e2755aade7f4 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 0337d014-7e9e-4db0-a4aa-05dfeea8efe7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 18b58bd2-d0c6-4dd5-9aaa-c4c925d6f40d kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: Scaled up replica set isvc-sklearn-graph-raw-3b643-predictor-584c5864d8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-sklearn-graph-raw-3b643-predictor.18a6e2755612fa5c namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 73f5ccce-98c3-4154-a26d-9a4789b3c9ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 1aa0b39c-32c6-4e96-9e71-8a8da9c82d29 kind: Event lastTimestamp: "2026-04-16T16:15:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:15:11Z" name: isvc-sklearn-graph-raw-3b643-predictor.18a6e278d495e78a namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: e909a548-1322-4cbc-b97d-57ac969f14dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 1aa0b39c-32c6-4e96-9e71-8a8da9c82d29 kind: Event lastTimestamp: "2026-04-16T16:15:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:15:11Z" name: isvc-sklearn-graph-raw-3b643-predictor.18a6e278d4f6d203 namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 9310acbc-b169-4765-b31f-2c221038be8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:15:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 1aa0b39c-32c6-4e96-9e71-8a8da9c82d29 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:15:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: isvc-sklearn-graph-raw-3b643-predictor.18a6e27c53fca428 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: ef59e6d2-ba7d-4a5b-b463-661993174a53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:15:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 1aa0b39c-32c6-4e96-9e71-8a8da9c82d29 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:15:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: isvc-sklearn-graph-raw-3b643-predictor.18a6e27c54638bff namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 4aa55692-bff3-4278-b413-8bacb40ed0a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 794fe78c-913a-4ff7-91b3-7c3d14bdffc5 kind: Event lastTimestamp: "2026-04-16T16:16:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:13Z" name: isvc-sklearn-graph-raw-3b643.18a6e27555efcaa3 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 6f8e9e6d-db94-4ede-bc00-c97e7214f5e9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 794fe78c-913a-4ff7-91b3-7c3d14bdffc5 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-3b643": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-3b643": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-sklearn-graph-raw-3b643.18a6e27556540c6f namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: 59aa36fe-6a54-4de5-aa7a-bdda063dcfbe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 794fe78c-913a-4ff7-91b3-7c3d14bdffc5 kind: Event lastTimestamp: "2026-04-16T16:16:13Z" message: InferenceService [isvc-sklearn-graph-raw-3b643] is Ready metadata: creationTimestamp: "2026-04-16T16:16:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:13Z" name: isvc-sklearn-graph-raw-3b643.18a6e2873fcfd236 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 344acbf1-108b-4079-bbb5-d59d89a039db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:16:37.044125Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28cb7611c0d namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 77e3b3e4-feeb-4dbc-a211-25892ccbc584 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28cd118b1a6 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: cf462d87-07ca-4be6-b16c-52b65a2491d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28cd2750bdb namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 4b19ac09-f59c-4d9f-b7b5-72c687fde8cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28cd60198b1 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 7d583e1d-e25c-4983-afa0-28a980a1293d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28cd6798620 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: df7d23a3-2f48-4c4d-b9d6-4c901faf94ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28dd3f58765 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: b3cdbd49-2a15-43ca-a62d-6401c016ce90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28dd82f8ea5 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 15529727-8db7-421f-9dad-e5624309b2d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28dd8a5b336 namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: c0833f5b-c2ae-4545-9a60-5b69db5e9ead reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:16:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:17:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:17:43Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e28e101d1673 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 63040b51-e822-48ef-9fc7-65d04b5bb76a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bac4d96d-acc8-46c7-8f87-1f241d316e33 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:17Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq.18a6e2a40d221bd1 namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 4e62f0aa-19d6-4b28-8b59-73cf27cb8b9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 78c32fb3-7283-4ec5-900f-3d174ae2eb33 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d-xwcpq' metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:37Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d.18a6e28cb6f1f562 namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 569b2029-9905-4212-b120-0b4590014bdc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 95173f04-3c32-4535-9cfd-d3990fdeb1f9 kind: Event lastTimestamp: "2026-04-16T16:16:36Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-724bc-predictor-795997bf5d from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:36Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor.18a6e28cb1244509 namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: 7ab41240-4b4d-4b52-8a84-7a74844903c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:16:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 4b8e6714-1b42-4e80-864c-e20ad06a8bc5 kind: Event lastTimestamp: "2026-04-16T16:17:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:17:21Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor.18a6e2902f851bbf namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 44e7fa5f-d207-4977-8cc8-527213c8498a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:16:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 4b8e6714-1b42-4e80-864c-e20ad06a8bc5 kind: Event lastTimestamp: "2026-04-16T16:17:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:17:21Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor.18a6e2902ff25920 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: b93924c2-38dd-4c4d-85f1-8c5fbbb1fee1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:17:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 4b8e6714-1b42-4e80-864c-e20ad06a8bc5 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor.18a6e29aacb6cb45 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: e8e225a8-1723-49ae-9062-fbcbe7ef3984 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 4b8e6714-1b42-4e80-864c-e20ad06a8bc5 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: isvc-sklearn-graph-raw-hpa-724bc-predictor.18a6e29aad1c19f3 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 37ad39dc-30db-44d2-84c8-b145d1fbdcac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:16:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 7bf70a10-f647-4414-975d-b42cd22ecbbe kind: Event lastTimestamp: "2026-04-16T16:17:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:17:53Z" name: isvc-sklearn-graph-raw-hpa-724bc.18a6e28cb08271ff namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: eeb1f043-c627-43f3-b1e0-cb459dc78329 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 7bf70a10-f647-4414-975d-b42cd22ecbbe kind: Event lastTimestamp: "2026-04-16T16:16:36Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-724bc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-724bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:36Z" name: isvc-sklearn-graph-raw-hpa-724bc.18a6e28cb12f9d86 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: ceb9eb40-76f7-414d-9afa-e52776f8682c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:17:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7bf70a10-f647-4414-975d-b42cd22ecbbe kind: Event lastTimestamp: "2026-04-16T16:17:53Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-724bc] is Ready metadata: creationTimestamp: "2026-04-16T16:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:17:53Z" name: isvc-sklearn-graph-raw-hpa-724bc.18a6e29e9c36aa35 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 68b157ad-86c2-40d5-aff1-1b5977180aa6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:19:52.539160Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17517" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2ba3bca98ff namespace: kserve-ci-e2e-test resourceVersion: "17524" uid: ba1253b3-854f-417d-b716-63c8604fff40 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2ba54bb27a7 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 65883240-6e82-480a-be5d-b1f0325528f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2ba561146f2 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 5fbe3cd7-034c-4a4e-91a3-372bc9fd8389 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:53Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2ba59040a75 namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: 82d028b2-3976-472b-b6d1-a1a0abce7b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:19:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:53Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2ba597fbcc2 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 56c6b954-69c3-4e76-9e64-1530e2e7188d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:57Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2bb6116f5c6 namespace: kserve-ci-e2e-test resourceVersion: "17601" uid: 47b83d30-3f37-4cb3-ba50-8f9cb9cd586f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:57Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2bb642b9b54 namespace: kserve-ci-e2e-test resourceVersion: "17606" uid: 92a481c9-af30-4e41-b1f2-fd7f254b1511 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:19:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:57Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2bb64a2712f namespace: kserve-ci-e2e-test resourceVersion: "17607" uid: c06ff4ed-b3e4-4b94-a5eb-33a4424b1db1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-16T16:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:22:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:26Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2bb9cf879b2 namespace: kserve-ci-e2e-test resourceVersion: "18726" uid: f4f4fbad-d23a-43e7-9587-16d8880a63b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: bc7aea67-0135-4660-ae60-28847adb7afe kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:22:22Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g.18a6e2dd325d007b namespace: kserve-ci-e2e-test resourceVersion: "18668" uid: 25a93380-3465-413a-a615-637c68f7a807 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-954a2-predictor-877cc765 namespace: kserve-ci-e2e-test resourceVersion: "17494" uid: 096ddf26-bf16-42b3-8e06-2f821f57b0ab kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: 'Created pod: isvc-sklearn-scale-raw-954a2-predictor-877cc765-jf86g' metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2-predictor-877cc765.18a6e2ba3b3b04f7 namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 8baee700-f76b-4237-899d-977199754ff1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-954a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17493" uid: 1d787939-8126-4b0b-bdec-66c20e9bf1f0 kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Scaled up replica set isvc-sklearn-scale-raw-954a2-predictor-877cc765 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2-predictor.18a6e2ba38503149 namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: 59e45529-0f25-4c4f-ad5c-f521af842b99 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:20:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-954a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: 3850257b-4f14-46d0-b601-11ec3f6fa84f kind: Event lastTimestamp: "2026-04-16T16:20:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:20:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:20:37Z" name: isvc-sklearn-scale-raw-954a2-predictor.18a6e2bdb6ff963a namespace: kserve-ci-e2e-test resourceVersion: "17897" uid: 8dfe31bc-fdb7-4819-a044-61870069a038 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:20:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-954a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: 3850257b-4f14-46d0-b601-11ec3f6fa84f kind: Event lastTimestamp: "2026-04-16T16:20:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:20:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:20:37Z" name: isvc-sklearn-scale-raw-954a2-predictor.18a6e2bdb75d7318 namespace: kserve-ci-e2e-test resourceVersion: "17898" uid: 807d6199-241a-47c2-ab25-3efc86181caf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:20:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-954a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: 3850257b-4f14-46d0-b601-11ec3f6fa84f kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-sklearn-scale-raw-954a2-predictor.18a6e2c8344f8a93 namespace: kserve-ci-e2e-test resourceVersion: "18642" uid: c0bbb17d-8489-42e2-8559-802748eecd8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:20:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-954a2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: 3850257b-4f14-46d0-b601-11ec3f6fa84f kind: Event lastTimestamp: "2026-04-16T16:22:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:22:22Z" name: isvc-sklearn-scale-raw-954a2-predictor.18a6e2c834baf43c namespace: kserve-ci-e2e-test resourceVersion: "18643" uid: e2a86c52-a3ea-4a6e-add9-ad32fe4cdcc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-954a2 namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: c5f3bea8-5783-44e4-9381-37e6d6dbf63f kind: Event lastTimestamp: "2026-04-16T16:22:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:22:16Z" name: isvc-sklearn-scale-raw-954a2.18a6e2ba3869642c namespace: kserve-ci-e2e-test resourceVersion: "18593" uid: b8b763ea-7b8c-47db-9396-17782a472ee3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-954a2 namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: c5f3bea8-5783-44e4-9381-37e6d6dbf63f kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-954a2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-954a2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:19:52Z" name: isvc-sklearn-scale-raw-954a2.18a6e2ba3bd707f7 namespace: kserve-ci-e2e-test resourceVersion: "17525" uid: 8f1861b0-b387-4f62-bf55-9c47180e7442 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-954a2 namespace: kserve-ci-e2e-test resourceVersion: "18591" uid: c5f3bea8-5783-44e4-9381-37e6d6dbf63f kind: Event lastTimestamp: "2026-04-16T16:22:16Z" message: InferenceService [isvc-sklearn-scale-raw-954a2] is Ready metadata: creationTimestamp: "2026-04-16T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:22:16Z" name: isvc-sklearn-scale-raw-954a2.18a6e2dbbe1d35d0 namespace: kserve-ci-e2e-test resourceVersion: "18592" uid: fa7cb77b-a93e-41b1-ba5f-2c6f24736ce6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:14:56.761192Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:14:56Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e2755e0d0b79 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 42a87245-a932-4b37-8e31-136f50dd933f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:14:57Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e2757800c427 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: fb9c5b67-8463-4d3b-bd17-756189e95f36 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e275797371bc namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 1318a643-24e3-4b7d-96a4-f0695bb4d3c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e2757c4d1f9d namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 845096bb-eec0-43d5-b868-e0d9b49e952c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:14:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:14:57Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e2757cb7c2f8 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 9e9cf0b4-c85e-4902-9dc7-12a3d3fb3f49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:15:01Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:01Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e2767754ead3 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: d78d97c1-f96f-438a-80ba-82d2ecdd4fdb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:15:18Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.074s (17.074s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T16:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:18Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e27a7114ce66 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 809012d5-851b-4d43-b7b2-75f08b8de680 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:15:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:18Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e27a7567c377 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: f62dd12d-65bc-495d-9193-ecc58d23e44d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:15:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:15:18Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e27a763edfe0 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: bc1a659c-93ae-4c37-9ce6-6c6e9820417c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:15:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:16:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:40Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e27aac7b368d namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 87cbfb5b-07e2-4193-97f6-1fa9347a8a8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 1078936f-0170-46f2-abbe-b429e8a45718 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s.18a6e28cbac6ffb6 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 9a3fd679-5984-4be5-86e2-de11eda38815 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-3b643-predictor-94775489 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: de12ab2a-3faf-4868-9660-e0f2196e1b06 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'Created pod: isvc-xgboost-graph-raw-3b643-predictor-94775489-88l6s' metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-xgboost-graph-raw-3b643-predictor-94775489.18a6e2755d8bbeba namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 63bc0781-5822-4bb9-b9c9-3e8aba1ebb65 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 62cf8857-037f-4d8b-9db4-f9f7c258d305 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: Scaled up replica set isvc-xgboost-graph-raw-3b643-predictor-94775489 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-xgboost-graph-raw-3b643-predictor.18a6e2755a7d1c2d namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 5ef20a97-fc1e-4810-914c-18877ddee626 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: f09168a6-fbff-4a71-a812-f8e60e867e75 kind: Event lastTimestamp: "2026-04-16T16:15:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:15:41Z" name: isvc-xgboost-graph-raw-3b643-predictor.18a6e278d8f5f373 namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 466c189a-ec88-4563-8da0-b4122ed71a75 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: f09168a6-fbff-4a71-a812-f8e60e867e75 kind: Event lastTimestamp: "2026-04-16T16:15:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:15:41Z" name: isvc-xgboost-graph-raw-3b643-predictor.18a6e278d95c3e41 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: ce3da488-124b-4011-8dae-0d8bf34d0230 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:15:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: f09168a6-fbff-4a71-a812-f8e60e867e75 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: isvc-xgboost-graph-raw-3b643-predictor.18a6e28355a42ef8 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: e297c141-8d70-45fc-807f-059959508d38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:15:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b643-predictor namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: f09168a6-fbff-4a71-a812-f8e60e867e75 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: isvc-xgboost-graph-raw-3b643-predictor.18a6e28355ed0b33 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 177d5a58-ea8e-49ff-a303-5138201e1295 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: fabd1e82-2299-4d20-bed1-caa68be41620 kind: Event lastTimestamp: "2026-04-16T16:16:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:20Z" name: isvc-xgboost-graph-raw-3b643.18a6e2755a5b8756 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 7132a57b-f041-4990-9693-d8006f190e37 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: fabd1e82-2299-4d20-bed1-caa68be41620 kind: Event lastTimestamp: "2026-04-16T16:14:56Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-3b643": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-3b643": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:14:56Z" name: isvc-xgboost-graph-raw-3b643.18a6e2755ae4ff79 namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: 515f30c3-c0af-4e1a-b47d-1645d0de5d86 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: fabd1e82-2299-4d20-bed1-caa68be41620 kind: Event lastTimestamp: "2026-04-16T16:16:20Z" message: InferenceService [isvc-xgboost-graph-raw-3b643] is Ready metadata: creationTimestamp: "2026-04-16T16:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:20Z" name: isvc-xgboost-graph-raw-3b643.18a6e288e4d44dd8 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 304887ae-ce67-4ffe-85ff-dd43c712bc4d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:16:37.089784Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28cba19cf4e namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 9f639bd3-a168-4f20-9e48-9f3951665354 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28cd4a492cb namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 09b43f1b-7b7a-4c94-af5f-770621773a73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28cd5d4d22b namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: fc2dc39d-db89-4ac7-ace1-e1464aa81eb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28cd97fb250 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 5e97d540-4d0a-4482-aa54-b835ae9838ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28cd9ef59b1 namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: d83ac483-76b1-4373-8607-a5d47f30d8a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28dd4240c1c namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: aad18824-f55b-4c36-a481-30345813ad3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28dd8183cc2 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: aafb6e7b-1d0b-46ac-9ad0-7208913b1e0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:16:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:41Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28dd8d326af namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 315cdd14-5f87-441b-ae33-d9d07a86bd85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:16:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:17:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:17:33Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e28e103776b7 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 60cccc9d-00c8-4196-bc9c-2b9d975b3418 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 72c04038-8cbe-40f0-b00f-35632444d276 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:17Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8.18a6e2a4130b3bb6 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: ab677e8f-9a1e-4752-8f09-333d48508b4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: a022b0d5-9408-420d-8981-e7e6ea9812f3 kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7-nglk8' metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7.18a6e28cb9ba0f3f namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 0524a05a-fa45-4f17-b8f0-625d6e91d92e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: de48337a-3c2a-49d3-b784-f0179d83c48a kind: Event lastTimestamp: "2026-04-16T16:16:37Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-724bc-predictor-8485c796b7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:37Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor.18a6e28cb6c1c0af namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 0f751b58-aa81-49b4-97a8-2ba2d07c8841 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:16:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 490da49d-6a79-4744-a758-18235bfb839a kind: Event lastTimestamp: "2026-04-16T16:17:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:17:22Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor.18a6e29035947ea9 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 3e8fe74b-7653-4c4d-b984-772a39a23b56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:16:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 490da49d-6a79-4744-a758-18235bfb839a kind: Event lastTimestamp: "2026-04-16T16:17:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:17:22Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor.18a6e2903604404a namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: e1efe6ac-5507-4d8d-933b-d8212c20cea4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 490da49d-6a79-4744-a758-18235bfb839a kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor.18a6e29ab22de5bc namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 6b25e489-83eb-47fb-a3f1-e6ab640aeb0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-724bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 490da49d-6a79-4744-a758-18235bfb839a kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: isvc-xgboost-graph-raw-hpa-724bc-predictor.18a6e29ab2909ce6 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 43690a60-8288-4404-860b-ce880e8d3bcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T16:16:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 03929986-8077-4629-bb90-d1e0da058fd6 kind: Event lastTimestamp: "2026-04-16T16:17:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:17:43Z" name: isvc-xgboost-graph-raw-hpa-724bc.18a6e28cb6f7cd5f namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 4efcf144-67e1-4e86-ad52-c4e48bc80a2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 03929986-8077-4629-bb90-d1e0da058fd6 kind: Event lastTimestamp: "2026-04-16T16:17:43Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-724bc] is Ready metadata: creationTimestamp: "2026-04-16T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:17:43Z" name: isvc-xgboost-graph-raw-hpa-724bc.18a6e29c4849e73e namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 12c63294-4567-495c-ad63-aaf3c1b434e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:18:17.277233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-74f70-predictor-767887b594-gh5xm to ip-10-0-129-182.ec2.internal metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a40dbcec09 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 56758daf-3bb1-4f6e-a9a6-0232f8e56c14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a415b49a66 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 2dc1af67-1616-4910-b162-1aed743d2434 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a416dac13c namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: dc69a4e1-7153-4271-9b0c-b4eeead991c1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:18:18Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 939ms (939ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-16T16:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:18Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a44ed8cd3b namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: a925c4f1-bc72-4f69-b833-8963e0521a2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:18:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:18Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a451f35d67 namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: 671b106e-c68f-4606-bb2b-3582cd0fd6b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:18:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:18Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2a45267385f namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: dbc9b2af-dbf9-4016-a299-3163abaeca04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-74f70-predictor-767887b594-gh5xm namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: cf02fbfe-8138-493d-92ae-dba214ad3704 kind: Event lastTimestamp: "2026-04-16T16:19:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:19:52Z" name: message-dumper-raw-74f70-predictor-767887b594-gh5xm.18a6e2ba42846ca9 namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: d9f061f5-b51e-4411-a371-5dfb04b4e05c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-74f70-predictor-767887b594 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 91c1814a-203f-4f9c-9655-8a3e4ebaaba5 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: 'Created pod: message-dumper-raw-74f70-predictor-767887b594-gh5xm' metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70-predictor-767887b594.18a6e2a40d56cfcf namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 74857dcc-a66f-42d7-a8da-b0d731235d59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 908815e8-09ec-494e-9cba-c92845f69ba8 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: Scaled up replica set message-dumper-raw-74f70-predictor-767887b594 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70-predictor.18a6e2a409f430d2 namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 8758b995-278d-4221-bb00-889192106e42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 22e387b2-ad59-409d-8b90-6c04357f739c kind: Event lastTimestamp: "2026-04-16T16:18:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:47Z" name: message-dumper-raw-74f70-predictor.18a6e2a788e9e5e4 namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: d9d0ed30-46f5-43af-be5a-b7bd109fa598 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-74f70-predictor namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 22e387b2-ad59-409d-8b90-6c04357f739c kind: Event lastTimestamp: "2026-04-16T16:18:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:47Z" name: message-dumper-raw-74f70-predictor.18a6e2a7896ba831 namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: 8ee43f34-84c4-414c-b319-e252858e848b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 8b691c5d-f5d2-4b80-9564-2453bbfe02e6 kind: Event lastTimestamp: "2026-04-16T16:18:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:18:19Z" name: message-dumper-raw-74f70.18a6e2a409f6a5cd namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 5834752d-ceff-4041-9f0a-a61b56c1e818 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 8b691c5d-f5d2-4b80-9564-2453bbfe02e6 kind: Event lastTimestamp: "2026-04-16T16:18:17Z" message: 'Failed to update status for InferenceService "message-dumper-raw-74f70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-74f70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:18:17Z" name: message-dumper-raw-74f70.18a6e2a40d78fd3b namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 30df0b45-c01c-4bd9-90df-a62a175167cd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-74f70 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 8b691c5d-f5d2-4b80-9564-2453bbfe02e6 kind: Event lastTimestamp: "2026-04-16T16:18:19Z" message: InferenceService [message-dumper-raw-74f70] is Ready metadata: creationTimestamp: "2026-04-16T16:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:18:19Z" name: message-dumper-raw-74f70.18a6e2a47973a7ac namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: e020a203-5b3d-4724-9f6b-879862844506 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:16:26.748170Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-3b643-7f87dff547-9r7xz to ip-10-0-129-182.ec2.internal metadata: creationTimestamp: "2026-04-16T16:16:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:16:26Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28a51b152ef namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: d163a6b5-6829-4189-a3e1-f7f0c9d49070 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:27Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:16:27Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28a6b5546f0 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: ea63f4d6-4bbb-467b-a7b7-ce08630c4b3b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:27Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" metadata: creationTimestamp: "2026-04-16T16:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:27Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28a6cb8d7c4 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 643890e8-1815-4286-ba26-da882c40d28c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" in 2.622s (2.622s including waiting). Image size: 216230482 bytes.' metadata: creationTimestamp: "2026-04-16T16:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:29Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28b090ea485 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 76994022-90ea-4f65-a186-8dcda8901543 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:30Z" message: 'Created container: model-chainer-raw-3b643' metadata: creationTimestamp: "2026-04-16T16:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:30Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28b0c1d5ce6 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: e0bf759e-7a10-4d10-bc4d-584549d89358 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:30Z" message: Started container model-chainer-raw-3b643 metadata: creationTimestamp: "2026-04-16T16:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:30Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28b0c7e6bed namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 96ab89ca-c38a-4d48-82fd-0e046bc54d5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" already present on machine metadata: creationTimestamp: "2026-04-16T16:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:30Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28b386a6e4d namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: f46a89ae-f5da-4605-aa85-ff83691cc51a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b643} kind: Pod name: model-chainer-raw-3b643-7f87dff547-9r7xz namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 2c95eb78-181b-4996-a4a5-b1f30325d319 kind: Event lastTimestamp: "2026-04-16T16:16:33Z" message: Back-off restarting failed container model-chainer-raw-3b643 in pod model-chainer-raw-3b643-7f87dff547-9r7xz_kserve-ci-e2e-test(2c95eb78-181b-4996-a4a5-b1f30325d319) metadata: creationTimestamp: "2026-04-16T16:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:16:33Z" name: model-chainer-raw-3b643-7f87dff547-9r7xz.18a6e28b74429aa2 namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 05081496-ea71-4866-8d76-c66ed8805c73 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-3b643-7f87dff547 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 1443d00e-fc89-4f09-8ae6-1b9de2f57cc4 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: 'Created pod: model-chainer-raw-3b643-7f87dff547-9r7xz' metadata: creationTimestamp: "2026-04-16T16:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: model-chainer-raw-3b643-7f87dff547.18a6e28a510ec3f5 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 2e446420-5ccd-4932-a857-cc8480ceae1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: f67cf8b4-6a24-4616-8ce2-a8d516eade76 kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: Scaled up replica set model-chainer-raw-3b643-7f87dff547 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:16:26Z" name: model-chainer-raw-3b643.18a6e28a4f9d7a4a namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: f4dcca76-e559-4b7f-8314-e69a54de6e03 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:16:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-3b643 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: f4bad1ad-dc64-45eb-b369-b6700974946f kind: Event lastTimestamp: "2026-04-16T16:16:26Z" message: InferenceGraph [model-chainer-raw-3b643] is Ready metadata: creationTimestamp: "2026-04-16T16:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:16:26Z" name: model-chainer-raw-3b643.18a6e28a558e3f4c namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 40ebb654-fa56-4d2e-a294-c96491967b91 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:18:07.049961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-724bc-d588cbf89-2tdkd to ip-10-0-129-182.ec2.internal metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a1ac25255e namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: e5671350-c4df-432b-918e-9b7dea60d1d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a1c553125c namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: b716ab71-82d4-4f32-adf7-70854fa071fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-724bc} kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7" already present on machine metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a1c69ff6f8 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 2de5a425-7a1e-48c4-b659-cdcd997f8ae6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-724bc} kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'Created container: model-chainer-raw-hpa-724bc' metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:08Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a1c99b9046 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 9c6b2f74-f95b-48fc-ba4a-bba514a4d51c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-724bc} kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: "2026-04-16T16:18:08Z" message: Started container model-chainer-raw-hpa-724bc metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:08Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a1ca0f9f25 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 2bb03306-dc0d-49ab-af8c-3d4f0dfcb477 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:18:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-724bc} kind: Pod name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 44c561b0-a63e-4057-8b7b-792102cd2f53 kind: Event lastTimestamp: "2026-04-16T16:18:12Z" message: Back-off restarting failed container model-chainer-raw-hpa-724bc in pod model-chainer-raw-hpa-724bc-d588cbf89-2tdkd_kserve-ci-e2e-test(44c561b0-a63e-4057-8b7b-792102cd2f53) metadata: creationTimestamp: "2026-04-16T16:18:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:18:12Z" name: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd.18a6e2a21dbf5a1e namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 6f660f89-daef-477f-9d9d-f2e2bb2dcf08 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-182.ec2.internal source: component: kubelet host: ip-10-0-129-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-724bc-d588cbf89 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 1db39598-6516-4b4f-8cbb-acb4dc09dcb2 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: 'Created pod: model-chainer-raw-hpa-724bc-d588cbf89-2tdkd' metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc-d588cbf89.18a6e2a1ab92ca51 namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: f05075c0-8ff9-499e-85f1-8809daa9b191 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 230252e9-1654-479f-bf8c-3cd83e82559b kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: Scaled up replica set model-chainer-raw-hpa-724bc-d588cbf89 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc.18a6e2a1aa75d67f namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: bdf17120-a8e0-414c-ab8c-0732874d359c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:18:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-724bc namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 0d0d5b7a-d668-4c29-9d6c-0204b6f0d397 kind: Event lastTimestamp: "2026-04-16T16:18:07Z" message: InferenceGraph [model-chainer-raw-hpa-724bc] is Ready metadata: creationTimestamp: "2026-04-16T16:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:18:07Z" name: model-chainer-raw-hpa-724bc.18a6e2a1b1000ac4 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: d20efa27-51c7-4547-b61b-9080a2623270 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:24:11.255130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-5512a-predictor-79fc6d66bc-djd6g to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f678771aff namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: a377c1a4-65ad-478b-8e63-ad71572120ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f691bc8cc2 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: c5acb723-a7db-4083-af38-5fa894e5733f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f693254d41 namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: e7339734-b4d1-4200-a075-1582bc4082ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f6961fe3c9 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: c5b99277-69d7-42d9-a369-c1301efc2a77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f69691b8d3 namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: bebe1675-4486-47d8-af29-a68842ccd841 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:16Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f7a547c71f namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 8d23bbb5-ddd0-4f94-98bb-a6e8cc2e2075 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:16Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f7a838a46d namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: a0e72a85-47c2-4644-b602-7fe93c32f2aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:24:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:24:16Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f7a8bbe492 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 4bbf2be9-54b6-4ac3-8581-b587d0144f65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:24:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:25:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:18Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e2f7e12ca621 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 10365118-cb03-4273-b064-c29f23ca62e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 19e85e6e-44d4-415b-aa6f-4bb4a8ddb850 kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g.18a6e30920b46ebb namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: ec669f59-8300-41cb-b72a-2d577a77fc6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-5512a-predictor-79fc6d66bc namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 0f5bc836-1fd4-46d0-bedc-a90cfef2f69b kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: 'Created pod: raw-sklearn-5512a-predictor-79fc6d66bc-djd6g' metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor-79fc6d66bc.18a6e2f677ebc5a6 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: da042145-693f-44f4-b332-4e909d421793 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-5512a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 14891749-5c80-4db2-b073-f8d297c80116 kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: Scaled up replica set raw-sklearn-5512a-predictor-79fc6d66bc from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a-predictor.18a6e2f675ab4e93 namespace: kserve-ci-e2e-test resourceVersion: "19681" uid: ddcd606e-f62d-41ae-a0af-43621bd25ebd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5512a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 9a8151ff-f3e7-48d9-ba63-b46a5c5854b7 kind: Event lastTimestamp: "2026-04-16T16:24:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:41Z" name: raw-sklearn-5512a-predictor.18a6e2f9f49ea62e namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 81e20166-ef7c-4463-beb0-0c18b0cc99da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5512a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 9a8151ff-f3e7-48d9-ba63-b46a5c5854b7 kind: Event lastTimestamp: "2026-04-16T16:24:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:24:41Z" name: raw-sklearn-5512a-predictor.18a6e2f9f50691e1 namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 52a7abed-f412-4078-a895-deaebe50bee0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5512a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 9a8151ff-f3e7-48d9-ba63-b46a5c5854b7 kind: Event lastTimestamp: "2026-04-16T16:25:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:25:26Z" name: raw-sklearn-5512a-predictor.18a6e300f2bcd56d namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: 996b96a2-be59-438d-9a96-a4f5f40343b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5512a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 9a8151ff-f3e7-48d9-ba63-b46a5c5854b7 kind: Event lastTimestamp: "2026-04-16T16:25:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:25:26Z" name: raw-sklearn-5512a-predictor.18a6e300f3262cff namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8debb827-d28c-4d25-bdad-d81214238572 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5512a namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: e07229d1-fe0b-46bf-9fb7-62a6145bbf5d kind: Event lastTimestamp: "2026-04-16T16:25:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:25:28Z" name: raw-sklearn-5512a.18a6e2f6758599f8 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: e98e2605-a37f-4a7e-9522-1c9b46d3a9b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5512a namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: e07229d1-fe0b-46bf-9fb7-62a6145bbf5d kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: 'Failed to update status for InferenceService "raw-sklearn-5512a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-5512a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a.18a6e2f675fe09d1 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: c8574481-5b90-4e73-865e-b9281b0f81b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:24:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5512a namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: e07229d1-fe0b-46bf-9fb7-62a6145bbf5d kind: Event lastTimestamp: "2026-04-16T16:24:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-5512a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:24:11Z" name: raw-sklearn-5512a.18a6e2f67bcf069b namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: a271a45a-dcf1-432a-afde-12575d8297a6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5512a namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: e07229d1-fe0b-46bf-9fb7-62a6145bbf5d kind: Event lastTimestamp: "2026-04-16T16:25:28Z" message: InferenceService [raw-sklearn-5512a] is Ready metadata: creationTimestamp: "2026-04-16T16:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:25:28Z" name: raw-sklearn-5512a.18a6e3086d322be7 namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 58b29517-d684-41da-bb02-ac6952f8a322 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:25:31.451031Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c to ip-10-0-135-144.ec2.internal metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e3092483537b namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 06eefd94-b972-4bad-a2f8-219f9003d964 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5b8dc579b9-tjhzc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e3093d82c76c namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: ff4ca34f-cf47-415c-ae08-095736bcf816 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840" already present on machine metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e3093efe1298 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 998c1ebf-ae76-4a9b-bdbf-3a403bae8caa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e30941e421d3 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: beada13d-ab6c-4235-adba-e85f89aa5b35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e309425666d7 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 570a4911-1420-4047-8d98-e67627100eb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326" already present on machine metadata: creationTimestamp: "2026-04-16T16:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:35Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e30a18ba8f53 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 11b894e5-12a4-4c9d-8191-2a3579762467 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:35Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e30a1b7ed0f6 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: ade1a4df-cf5a-4fb5-85a8-6fa5344e4402 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:25:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:25:35Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e30a1bf4d65a namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: aec56b95-1b2b-4cd1-b344-91026b7396ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T16:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:26:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:26:38Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e30a54b80564 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 7f00be92-d185-4dbf-86a4-fe236622d4ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: fada71ee-dd50-43ce-8b32-6c2ae09e643a kind: Event lastTimestamp: "2026-04-16T16:26:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:26:51Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c.18a6e31bce0e81ac namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: e50a2ce2-e731-4859-a135-d7e2185ffa90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-144.ec2.internal source: component: kubelet host: ip-10-0-135-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-82bab-predictor-c74656cbf namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: e01c3f03-ccc2-4d5e-8561-e07392b211cf kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: 'Created pod: raw-sklearn-runtime-82bab-predictor-c74656cbf-cdn6c' metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor-c74656cbf.18a6e309240cdee1 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a403ebcc-6c4e-4ba0-99bd-4a3b5ee772ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-82bab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 4f412f91-3ec5-490c-923f-aafcf41a6f3b kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: Scaled up replica set raw-sklearn-runtime-82bab-predictor-c74656cbf from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab-predictor.18a6e30921d22963 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 305d60be-cbb8-477b-a655-fb3dbfae2c2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-82bab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 3fc96f78-39e8-4b13-8899-555182c6baea kind: Event lastTimestamp: "2026-04-16T16:26:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:26:16Z" name: raw-sklearn-runtime-82bab-predictor.18a6e30ca048ffd9 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: a8cd17d5-8176-4aa9-a606-fab504e0b5e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-82bab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 3fc96f78-39e8-4b13-8899-555182c6baea kind: Event lastTimestamp: "2026-04-16T16:26:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:26:16Z" name: raw-sklearn-runtime-82bab-predictor.18a6e30ca08ce575 namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: c978b38d-6e73-4e5d-b84b-2f8e18bd91a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:26:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-82bab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 3fc96f78-39e8-4b13-8899-555182c6baea kind: Event lastTimestamp: "2026-04-16T16:26:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:26:46Z" name: raw-sklearn-runtime-82bab-predictor.18a6e3171de2f19c namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 6cfacb2c-d7d6-4776-b8dc-fe641ae56bf2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:26:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-82bab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 3fc96f78-39e8-4b13-8899-555182c6baea kind: Event lastTimestamp: "2026-04-16T16:26:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:26:46Z" name: raw-sklearn-runtime-82bab-predictor.18a6e3171e4c7b3c namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 9a40aff8-529f-4807-83bc-33b04b22b202 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-82bab namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 15de1ded-19fc-48f6-8dab-43ea71f22f41 kind: Event lastTimestamp: "2026-04-16T16:26:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:26:48Z" name: raw-sklearn-runtime-82bab.18a6e309219f1bc9 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 1062e327-887e-49dc-9b2c-472211d9e2ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-82bab namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 15de1ded-19fc-48f6-8dab-43ea71f22f41 kind: Event lastTimestamp: "2026-04-16T16:25:31Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-82bab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-82bab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:25:31Z" name: raw-sklearn-runtime-82bab.18a6e3092200c1fa namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 0ec5d7fa-28b3-4b8f-8e4d-cf3d442da14e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:26:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-82bab namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 15de1ded-19fc-48f6-8dab-43ea71f22f41 kind: Event lastTimestamp: "2026-04-16T16:26:48Z" message: InferenceService [raw-sklearn-runtime-82bab] is Ready metadata: creationTimestamp: "2026-04-16T16:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:26:48Z" name: raw-sklearn-runtime-82bab.18a6e31b11622649 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: a0663de6-a29d-43bd-bb32-36d9bb9d61eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21156"